diff --git a/error.py b/error.py index 225eb59d..8bb64b8f 100644 --- a/error.py +++ b/error.py @@ -13,6 +13,10 @@ # limitations under the License. +# URL to file bug reports for repo tool issues. +BUG_REPORT_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue' + + class ManifestParseError(Exception): """Failed to parse the manifest file. """ diff --git a/git_superproject.py b/git_superproject.py new file mode 100644 index 00000000..3e87e929 --- /dev/null +++ b/git_superproject.py @@ -0,0 +1,149 @@ +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provide functionality to get all projects and their SHAs from Superproject. + +For more information on superproject, check out: +https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects + +Examples: + superproject = Superproject() + project_shas = superproject.GetAllProjectsSHAs() +""" + +import os +import sys + +from error import GitError +from git_command import GitCommand +import platform_utils + + +class Superproject(object): + """Get SHAs from superproject. + + It does a 'git clone' of superproject and 'git ls-tree' to get list of SHAs for all projects. + It contains project_shas which is a dictionary with project/sha entries. + """ + def __init__(self, repodir, superproject_dir='exp-superproject'): + """Initializes superproject. + + Args: + repodir: Path to the .repo/ dir for holding all internal checkout state. + superproject_dir: Relative path under |repodir| to checkout superproject. + """ + self._project_shas = None + self._repodir = os.path.abspath(repodir) + self._superproject_dir = superproject_dir + self._superproject_path = os.path.join(self._repodir, superproject_dir) + + @property + def project_shas(self): + """Returns a dictionary of projects and their SHAs.""" + return self._project_shas + + def _Clone(self, url, branch=None): + """Do a 'git clone' for the given url and branch. + + Args: + url: superproject's url to be passed to git clone. + branch: the branchname to be passed as argument to git clone. + + Returns: + True if 'git clone ' is successful, or False. + """ + cmd = ['clone', url, '--depth', '1'] + if branch: + cmd += ['--branch', branch] + p = GitCommand(None, + cmd, + cwd=self._superproject_path, + capture_stdout=True, + capture_stderr=True) + retval = p.Wait() + if retval: + # `git clone` is documented to produce an exit status of `128` if + # the requested url or branch are not present in the configuration. + print('repo: error: git clone call failed with return code: %r, stderr: %r' % + (retval, p.stderr), file=sys.stderr) + return False + return True + + def _LsTree(self): + """Returns the data from 'git ls-tree -r HEAD'. + + Works only in git repositories. + + Returns: + data: data returned from 'git ls-tree -r HEAD' instead of None. + """ + git_dir = os.path.join(self._superproject_path, 'superproject') + if not os.path.exists(git_dir): + raise GitError('git ls-tree. Missing drectory: %s' % git_dir) + data = None + cmd = ['ls-tree', '-z', '-r', 'HEAD'] + p = GitCommand(None, + cmd, + cwd=git_dir, + capture_stdout=True, + capture_stderr=True) + retval = p.Wait() + if retval == 0: + data = p.stdout + else: + # `git clone` is documented to produce an exit status of `128` if + # the requested url or branch are not present in the configuration. + print('repo: error: git ls-tree call failed with return code: %r, stderr: %r' % ( + retval, p.stderr), file=sys.stderr) + return data + + def GetAllProjectsSHAs(self, url, branch=None): + """Get SHAs for all projects from superproject and save them in _project_shas. + + Args: + url: superproject's url to be passed to git clone. + branch: the branchname to be passed as argument to git clone. + + Returns: + A dictionary with the projects/SHAs instead of None. + """ + if not url: + raise ValueError('url argument is not supplied.') + if os.path.exists(self._superproject_path): + platform_utils.rmtree(self._superproject_path) + os.mkdir(self._superproject_path) + + # TODO(rtenneti): we shouldn't be cloning the repo from scratch every time. + if not self._Clone(url, branch): + raise GitError('git clone failed for url: %s' % url) + + data = self._LsTree() + if not data: + raise GitError('git ls-tree failed for url: %s' % url) + + # Parse lines like the following to select lines starting with '160000' and + # build a dictionary with project path (last element) and its SHA (3rd element). + # + # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00 + # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 + shas = {} + for line in data.split('\x00'): + ls_data = line.split(None, 3) + if not ls_data: + break + if ls_data[0] == '160000': + shas[ls_data[3]] = ls_data[2] + + self._project_shas = shas + return shas diff --git a/project.py b/project.py index 6c6534d8..17c75b4d 100644 --- a/project.py +++ b/project.py @@ -1197,6 +1197,9 @@ class Project(object): raise ManifestInvalidRevisionError('revision %s in %s not found' % (self.revisionExpr, self.name)) + def SetRevisionId(self, revisionId): + self.revisionId = revisionId + def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False): """Perform only the local IO portion of the sync process. Network access is not required. diff --git a/subcmds/sync.py b/subcmds/sync.py index 3482946d..d6b8f9dc 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -51,11 +51,12 @@ import event_log from git_command import GIT, git_require from git_config import GetUrlCookieFile from git_refs import R_HEADS, HEAD +import git_superproject import gitc_utils from project import Project from project import RemoteSpec from command import Command, MirrorSafeCommand -from error import RepoChangedException, GitError, ManifestParseError +from error import BUG_REPORT_URL, RepoChangedException, GitError, ManifestParseError import platform_utils from project import SyncBuffer from progress import Progress @@ -241,6 +242,8 @@ later is required to fix a server side protocol bug. p.add_option('--fetch-submodules', dest='fetch_submodules', action='store_true', help='fetch submodules from server') + p.add_option('--use-superproject', action='store_true', + help='use the manifest superproject to sync projects') p.add_option('--no-tags', dest='tags', default=True, action='store_false', help="don't fetch tags") @@ -894,6 +897,41 @@ later is required to fix a server side protocol bug. missing_ok=True, submodules_ok=opt.fetch_submodules) + if opt.use_superproject: + if not self.manifest.superproject: + print('error: superproject tag is not defined in manifest.xml', + file=sys.stderr) + sys.exit(1) + print('WARNING: --use-superproject is experimental and not ' + 'for general use', file=sys.stderr) + superproject_url = self.manifest.superproject['remote'].url + if not superproject_url: + print('error: superproject URL is not defined in manifest.xml', + file=sys.stderr) + sys.exit(1) + superproject = git_superproject.Superproject(self.manifest.repodir) + try: + superproject_shas = superproject.GetAllProjectsSHAs(url=superproject_url) + except Exception as e: + print('error: Cannot get project SHAs for %s: %s: %s' % + (superproject_url, type(e).__name__, str(e)), + file=sys.stderr) + sys.exit(1) + projects_missing_shas = [] + for project in all_projects: + path = project.relpath + if not path: + continue + sha = superproject_shas.get(path) + if sha: + project.SetRevisionId(sha) + else: + projects_missing_shas.append(path) + if projects_missing_shas: + print('error: please file a bug using %s to report missing shas for: %s' % + (BUG_REPORT_URL, projects_missing_shas), file=sys.stderr) + sys.exit(1) + err_network_sync = False err_update_projects = False err_checkout = False diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py new file mode 100644 index 00000000..67a75a17 --- /dev/null +++ b/tests/test_git_superproject.py @@ -0,0 +1,82 @@ +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittests for the git_superproject.py module.""" + +import os +import tempfile +import unittest +from unittest import mock + +from error import GitError +import git_superproject +import platform_utils + + +class SuperprojectTestCase(unittest.TestCase): + """TestCase for the Superproject module.""" + + def setUp(self): + """Set up superproject every time.""" + self.tempdir = tempfile.mkdtemp(prefix='repo_tests') + self.repodir = os.path.join(self.tempdir, '.repo') + os.mkdir(self.repodir) + self._superproject = git_superproject.Superproject(self.repodir) + + def tearDown(self): + """Tear down superproject every time.""" + platform_utils.rmtree(self.tempdir) + + def test_superproject_get_project_shas_no_url(self): + """Test with no url.""" + with self.assertRaises(ValueError): + self._superproject.GetAllProjectsSHAs(url=None) + + def test_superproject_get_project_shas_invalid_url(self): + """Test with an invalid url.""" + with self.assertRaises(GitError): + self._superproject.GetAllProjectsSHAs(url='localhost') + + def test_superproject_get_project_shas_invalid_branch(self): + """Test with an invalid branch.""" + with self.assertRaises(GitError): + self._superproject.GetAllProjectsSHAs( + url='sso://android/platform/superproject', + branch='junk') + + def test_superproject_get_project_shas_mock_clone(self): + """Test with _Clone failing.""" + with self.assertRaises(GitError): + with mock.patch.object(self._superproject, '_Clone', return_value=False): + self._superproject.GetAllProjectsSHAs(url='localhost') + + def test_superproject_get_project_shas_mock_ls_tree(self): + """Test with LsTree being a mock.""" + data = ('120000 blob 158258bdf146f159218e2b90f8b699c4d85b5804\tAndroid.bp\x00' + '160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' + '160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00' + '120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00' + '160000 commit ade9b7a0d874e25fff4bf2552488825c6f111928\tbuild/bazel\x00') + with mock.patch.object(self._superproject, '_Clone', return_value=True): + with mock.patch.object(self._superproject, '_LsTree', return_value=data): + shas = self._superproject.GetAllProjectsSHAs(url='localhost', branch='junk') + self.assertEqual(shas, { + 'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea', + 'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06', + 'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928' + }) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py index e2c83af9..370eb4f5 100644 --- a/tests/test_manifest_xml.py +++ b/tests/test_manifest_xml.py @@ -232,6 +232,7 @@ class XmlManifestTests(unittest.TestCase): """) self.assertEqual(manifest.superproject['name'], 'superproject') self.assertEqual(manifest.superproject['remote'].name, 'test-remote') + self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject') self.assertEqual( manifest.ToXml().toxml(), '' + @@ -245,20 +246,21 @@ class XmlManifestTests(unittest.TestCase): manifest = self.getXmlManifest(""" - + - + """) - self.assertEqual(manifest.superproject['name'], 'superproject') - self.assertEqual(manifest.superproject['remote'].name, 'test-remote') + self.assertEqual(manifest.superproject['name'], 'platform/superproject') + self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote') + self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject') self.assertEqual( manifest.ToXml().toxml(), '' + '' + - '' + + '' + '' + - '' + + '' + '') def test_superproject_with_defalut_remote(self):