diff --git a/git_superproject.py b/git_superproject.py index e2045cfd..465d1f87 100644 --- a/git_superproject.py +++ b/git_superproject.py @@ -25,8 +25,9 @@ Examples: import os import sys -from error import GitError +from error import BUG_REPORT_URL, GitError from git_command import GitCommand +import platform_utils class Superproject(object): @@ -46,6 +47,9 @@ class Superproject(object): self._repodir = os.path.abspath(repodir) self._superproject_dir = superproject_dir self._superproject_path = os.path.join(self._repodir, superproject_dir) + self._manifest_path = os.path.join(self._superproject_path, + 'superproject_override.xml') + self._work_git = os.path.join(self._superproject_path, 'superproject') @property def project_shas(self): @@ -57,7 +61,7 @@ class Superproject(object): Args: url: superproject's url to be passed to git clone. - branch: the branchname to be passed as argument to git clone. + branch: The branchname to be passed as argument to git clone. Returns: True if 'git clone ' is successful, or False. @@ -86,13 +90,12 @@ class Superproject(object): Returns: True if 'git pull ' is successful, or False. """ - git_dir = os.path.join(self._superproject_path, 'superproject') - if not os.path.exists(git_dir): - raise GitError('git pull. Missing drectory: %s' % git_dir) + if not os.path.exists(self._work_git): + raise GitError('git pull missing drectory: %s' % self._work_git) cmd = ['pull'] p = GitCommand(None, cmd, - cwd=git_dir, + cwd=self._work_git, capture_stdout=True, capture_stderr=True) retval = p.Wait() @@ -110,14 +113,13 @@ class Superproject(object): Returns: data: data returned from 'git ls-tree -r HEAD' instead of None. """ - git_dir = os.path.join(self._superproject_path, 'superproject') - if not os.path.exists(git_dir): - raise GitError('git ls-tree. Missing drectory: %s' % git_dir) + if not os.path.exists(self._work_git): + raise GitError('git ls-tree. Missing drectory: %s' % self._work_git) data = None cmd = ['ls-tree', '-z', '-r', 'HEAD'] p = GitCommand(None, cmd, - cwd=git_dir, + cwd=self._work_git, capture_stdout=True, capture_stderr=True) retval = p.Wait() @@ -130,22 +132,26 @@ class Superproject(object): retval, p.stderr), file=sys.stderr) return data - def GetAllProjectsSHAs(self, url, branch=None): + def _GetAllProjectsSHAs(self, url, branch=None): """Get SHAs for all projects from superproject and save them in _project_shas. Args: - url: superproject's url to be passed to git clone. - branch: the branchname to be passed as argument to git clone. + url: superproject's url to be passed to git clone or pull. + branch: The branchname to be passed as argument to git clone or pull. Returns: A dictionary with the projects/SHAs instead of None. """ if not url: raise ValueError('url argument is not supplied.') + do_clone = True if os.path.exists(self._superproject_path): if not self._Pull(): - raise GitError('git pull failed for url: %s' % url) - else: + # If pull fails due to a corrupted git directory, then do a git clone. + platform_utils.rmtree(self._superproject_path) + else: + do_clone = False + if do_clone: if not self._Clone(url, branch): raise GitError('git clone failed for url: %s' % url) @@ -168,3 +174,67 @@ class Superproject(object): self._project_shas = shas return shas + + def _WriteManfiestFile(self, manifest): + """Writes manifest to a file. + + Args: + manifest: A Manifest object that is to be written to a file. + + Returns: + manifest_path: Path name of the file into which manifest is written instead of None. + """ + if not os.path.exists(self._superproject_path): + print('error: missing superproject directory %s' % + self._superproject_path, + file=sys.stderr) + return None + manifest_str = manifest.ToXml().toxml() + manifest_path = self._manifest_path + try: + with open(manifest_path, 'w', encoding='utf-8') as fp: + fp.write(manifest_str) + except IOError as e: + print('error: cannot write manifest to %s:\n%s' + % (manifest_path, e), + file=sys.stderr) + return None + return manifest_path + + def UpdateProjectsRevisionId(self, manifest, projects, url, branch=None): + """Update revisionId of every project in projects with the SHA. + + Args: + manifest: A Manifest object that is to be written to a file. + projects: List of projects whose revisionId needs to be updated. + url: superproject's url to be passed to git clone or fetch. + branch: The branchname to be passed as argument to git clone or pull. + + Returns: + manifest_path: Path name of the overriding manfiest file instead of None. + """ + try: + shas = self._GetAllProjectsSHAs(url=url, branch=branch) + except Exception as e: + print('error: Cannot get project SHAs for %s: %s: %s' % + (url, type(e).__name__, str(e)), + file=sys.stderr) + return None + + projects_missing_shas = [] + for project in projects: + path = project.relpath + if not path: + continue + sha = shas.get(path) + if sha: + project.SetRevisionId(sha) + else: + projects_missing_shas.append(path) + if projects_missing_shas: + print('error: please file a bug using %s to report missing shas for: %s' % + (BUG_REPORT_URL, projects_missing_shas), file=sys.stderr) + return None + + manifest_path = self._WriteManfiestFile(manifest) + return manifest_path diff --git a/manifest_xml.py b/manifest_xml.py index eb8a98cc..58c63246 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -403,6 +403,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md revision = self.remotes[p.remote.orig_name].revision or d.revisionExpr if not revision or revision != p.revisionExpr: e.setAttribute('revision', p.revisionExpr) + else: + if p.revisionId: + e.setAttribute('revision', p.revisionId) if (p.upstream and (p.upstream != p.revisionExpr or p.upstream != d.upstreamExpr)): e.setAttribute('upstream', p.upstream) diff --git a/subcmds/sync.py b/subcmds/sync.py index 225e565a..c0f605a8 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -56,7 +56,7 @@ import gitc_utils from project import Project from project import RemoteSpec from command import Command, MirrorSafeCommand -from error import BUG_REPORT_URL, RepoChangedException, GitError, ManifestParseError +from error import RepoChangedException, GitError, ManifestParseError import platform_utils from project import SyncBuffer from progress import Progress @@ -271,6 +271,47 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) + def _UpdateProjectsRevisionId(self, opt, args): + """Update revisionId of every project with the SHA from superproject. + + This function updates each project's revisionId with SHA from superproject. + It writes the updated manifest into a file and reloads the manifest from it. + + Args: + opt: Program options returned from optparse. See _Options(). + args: Arguments to pass to GetProjects. See the GetProjects + docstring for details. + + Returns: + Returns path to the overriding manifest file. + """ + if not self.manifest.superproject: + print('error: superproject tag is not defined in manifest.xml', + file=sys.stderr) + sys.exit(1) + print('WARNING: --use-superproject is experimental and not ' + 'for general use', file=sys.stderr) + + superproject_url = self.manifest.superproject['remote'].url + if not superproject_url: + print('error: superproject URL is not defined in manifest.xml', + file=sys.stderr) + sys.exit(1) + + superproject = git_superproject.Superproject(self.manifest.repodir) + all_projects = self.GetProjects(args, + missing_ok=True, + submodules_ok=opt.fetch_submodules) + manifest_path = superproject.UpdateProjectsRevisionId(self.manifest, + all_projects, + url=superproject_url) + if not manifest_path: + print('error: Update of revsionId from superproject has failed', + file=sys.stderr) + sys.exit(1) + self._ReloadManifest(manifest_path) + return manifest_path + def _FetchProjectList(self, opt, projects, sem, *args, **kwargs): """Main function of the fetch threads. @@ -859,6 +900,9 @@ later is required to fix a server side protocol bug. else: self._UpdateManifestProject(opt, mp, manifest_name) + if opt.use_superproject: + manifest_name = self._UpdateProjectsRevisionId(opt, args) + if self.gitc_manifest: gitc_manifest_projects = self.GetProjects(args, missing_ok=True) @@ -898,41 +942,6 @@ later is required to fix a server side protocol bug. missing_ok=True, submodules_ok=opt.fetch_submodules) - if opt.use_superproject: - if not self.manifest.superproject: - print('error: superproject tag is not defined in manifest.xml', - file=sys.stderr) - sys.exit(1) - print('WARNING: --use-superproject is experimental and not ' - 'for general use', file=sys.stderr) - superproject_url = self.manifest.superproject['remote'].url - if not superproject_url: - print('error: superproject URL is not defined in manifest.xml', - file=sys.stderr) - sys.exit(1) - superproject = git_superproject.Superproject(self.manifest.repodir) - try: - superproject_shas = superproject.GetAllProjectsSHAs(url=superproject_url) - except Exception as e: - print('error: Cannot get project SHAs for %s: %s: %s' % - (superproject_url, type(e).__name__, str(e)), - file=sys.stderr) - sys.exit(1) - projects_missing_shas = [] - for project in all_projects: - path = project.relpath - if not path: - continue - sha = superproject_shas.get(path) - if sha: - project.SetRevisionId(sha) - else: - projects_missing_shas.append(path) - if projects_missing_shas: - print('error: please file a bug using %s to report missing shas for: %s' % - (BUG_REPORT_URL, projects_missing_shas), file=sys.stderr) - sys.exit(1) - err_network_sync = False err_update_projects = False err_checkout = False diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py index 4012ec26..08e8a5bf 100644 --- a/tests/test_git_superproject.py +++ b/tests/test_git_superproject.py @@ -21,6 +21,7 @@ from unittest import mock from error import GitError import git_superproject +import manifest_xml import platform_utils @@ -31,27 +32,43 @@ class SuperprojectTestCase(unittest.TestCase): """Set up superproject every time.""" self.tempdir = tempfile.mkdtemp(prefix='repo_tests') self.repodir = os.path.join(self.tempdir, '.repo') - os.mkdir(self.repodir) self._superproject = git_superproject.Superproject(self.repodir) + self.manifest_file = os.path.join( + self.repodir, manifest_xml.MANIFEST_FILE_NAME) + os.mkdir(self.repodir) + + # The manifest parsing really wants a git repo currently. + gitdir = os.path.join(self.repodir, 'manifests.git') + os.mkdir(gitdir) + with open(os.path.join(gitdir, 'config'), 'w') as fp: + fp.write("""[remote "origin"] + url = https://localhost:0/manifest +""") def tearDown(self): """Tear down superproject every time.""" platform_utils.rmtree(self.tempdir) + def getXmlManifest(self, data): + """Helper to initialize a manifest for testing.""" + with open(self.manifest_file, 'w') as fp: + fp.write(data) + return manifest_xml.XmlManifest(self.repodir, self.manifest_file) + def test_superproject_get_project_shas_no_url(self): """Test with no url.""" with self.assertRaises(ValueError): - self._superproject.GetAllProjectsSHAs(url=None) + self._superproject._GetAllProjectsSHAs(url=None) def test_superproject_get_project_shas_invalid_url(self): """Test with an invalid url.""" with self.assertRaises(GitError): - self._superproject.GetAllProjectsSHAs(url='localhost') + self._superproject._GetAllProjectsSHAs(url='localhost') def test_superproject_get_project_shas_invalid_branch(self): """Test with an invalid branch.""" with self.assertRaises(GitError): - self._superproject.GetAllProjectsSHAs( + self._superproject._GetAllProjectsSHAs( url='sso://android/platform/superproject', branch='junk') @@ -59,14 +76,14 @@ class SuperprojectTestCase(unittest.TestCase): """Test with _Clone failing.""" with self.assertRaises(GitError): with mock.patch.object(self._superproject, '_Clone', return_value=False): - self._superproject.GetAllProjectsSHAs(url='localhost') + self._superproject._GetAllProjectsSHAs(url='localhost') def test_superproject_get_project_shas_mock_pull(self): """Test with _Pull failing.""" with self.assertRaises(GitError): with mock.patch.object(self._superproject, '_Clone', return_value=True): with mock.patch.object(self._superproject, '_Pull', return_value=False): - self._superproject.GetAllProjectsSHAs(url='localhost') + self._superproject._GetAllProjectsSHAs(url='localhost') def test_superproject_get_project_shas_mock_ls_tree(self): """Test with LsTree being a mock.""" @@ -77,13 +94,71 @@ class SuperprojectTestCase(unittest.TestCase): '160000 commit ade9b7a0d874e25fff4bf2552488825c6f111928\tbuild/bazel\x00') with mock.patch.object(self._superproject, '_Clone', return_value=True): with mock.patch.object(self._superproject, '_LsTree', return_value=data): - shas = self._superproject.GetAllProjectsSHAs(url='localhost', branch='junk') + shas = self._superproject._GetAllProjectsSHAs(url='localhost', branch='junk') self.assertEqual(shas, { 'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea', 'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06', 'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928' }) + def test_superproject_write_manifest_file(self): + """Test with writing manifest to a file after setting revisionId.""" + manifest = self.getXmlManifest(""" + + + + + +""") + self.assertEqual(len(manifest.projects), 1) + project = manifest.projects[0] + project.SetRevisionId('ABCDEF') + # Create temporary directory so that it can write the file. + os.mkdir(self._superproject._superproject_path) + manifest_path = self._superproject._WriteManfiestFile(manifest) + self.assertIsNotNone(manifest_path) + with open(manifest_path, "r") as fp: + manifest_xml = fp.read() + self.assertEqual( + manifest_xml, + '' + + '' + + '' + + '' + + '') + + def test_superproject_update_project_revision_id(self): + """Test with LsTree being a mock.""" + manifest = self.getXmlManifest(""" + + + + + +""") + self.assertEqual(len(manifest.projects), 1) + projects = manifest.projects + data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' + '160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00') + with mock.patch.object(self._superproject, '_Clone', return_value=True): + with mock.patch.object(self._superproject, '_Pull', return_value=True): + with mock.patch.object(self._superproject, '_LsTree', return_value=data): + # Create temporary directory so that it can write the file. + os.mkdir(self._superproject._superproject_path) + manifest_path = self._superproject.UpdateProjectsRevisionId( + manifest, projects, url='localhost') + self.assertIsNotNone(manifest_path) + with open(manifest_path, "r") as fp: + manifest_xml = fp.read() + self.assertEqual( + manifest_xml, + '' + + '' + + '' + + '' + + '') + if __name__ == '__main__': unittest.main() diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py index 370eb4f5..5fd242fc 100644 --- a/tests/test_manifest_xml.py +++ b/tests/test_manifest_xml.py @@ -327,6 +327,26 @@ class XmlManifestTests(unittest.TestCase): result['extras'], ['g1', 'g2', 'g1', 'name:extras', 'all', 'path:path']) + def test_project_set_revision_id(self): + """Check setting of project's revisionId.""" + manifest = self.getXmlManifest(""" + + + + + +""") + self.assertEqual(len(manifest.projects), 1) + project = manifest.projects[0] + project.SetRevisionId('ABCDEF') + self.assertEqual( + manifest.ToXml().toxml(), + '' + + '' + + '' + + '' + + '') + def test_include_levels(self): root_m = os.path.join(self.manifest_dir, 'root.xml') with open(root_m, 'w') as fp: