From 0125ae2fda18deee89dc94b32a2daa1b37a8a361 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 18:05:23 -0700 Subject: [PATCH] Introduce manifest format using git submodules If a manifest top level directory contains '.gitmodules' we now assume this is a git module format manifest and switch to using that code, rather than the legacy XML based manifest. At the same time, we move the bare repository for a project from $TOP/.repo/projects/$REPO_PATH.git to be $REPO_NAME.git instead. This makes it easier for us to later support a repo init from an existing work tree, as we can more accurately predict the path of the project's repository in the workspace. It also means that the $TOP/.repo/projects/ directory is layed out like a mirror would be. Signed-off-by: Shawn O. Pearce --- docs/manifest_submodule.txt | 130 ++++++++++ manifest_loader.py | 3 + manifest_submodule.py | 474 ++++++++++++++++++++++++++++++++++++ subcmds/init.py | 9 + subcmds/manifest.py | 4 +- 5 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 docs/manifest_submodule.txt create mode 100644 manifest_submodule.py diff --git a/docs/manifest_submodule.txt b/docs/manifest_submodule.txt new file mode 100644 index 00000000..e7d1f643 --- /dev/null +++ b/docs/manifest_submodule.txt @@ -0,0 +1,130 @@ +repo Manifest Format (submodule) +================================ + +A repo manifest describes the structure of a repo client; that is +the directories that are visible and where they should be obtained +from with git. + +The basic structure of a manifest is a bare Git repository holding +a 'gitmodules' file in the top level directory, and one or more +gitlink references pointing at commits from the referenced projects. +This is the same structure as used by 'git submodule'. + +Manifests are inherently version controlled, since they are kept +within a Git repository. Updates to manifests are automatically +obtained by clients during `repo sync`. + +.gitmodules +=========== + +The '.gitmodules' file, located in the top-level directory of the +client's working tree (or manifest repository), is a text file with +a syntax matching the requirements of 'git config'. + +This file contains one subsection per project (also called a +submodule by git), and the subsection value is a unique name to +describe the project. Each submodule section must contain the +following required keys: + + * path + * url + +submodule..path +--------------------- + +Defines the path, relative to the top-level directory of the client's +working tree, where the project is expected to be checked out. The +path name must not end with a '/'. All paths must be unique within +the .gitmodules file. + +At the specified path within the manifest repository a gitlink +tree entry (an entry with file mode 160000) must exist referencing +a commit SHA-1 from the project. This tree entry specifies the +exact version of the project that `repo sync` will synchronize the +client's working tree to. + +submodule..url +-------------------- + +Defines a URL from where the project repository can be cloned. +By default `repo sync` will clone from this URL whenever a user +needs to access this project. + +submodule..revision +------------------------- + +Name of the branch in the project repository that Gerrit Code Review +should automatically refresh the project's gitlink entry from. + +If set, during submit of a change within the referenced project, +Gerrit Code Review will automatically update the manifest +repository's corresponding gitlink to the new commit SHA-1 of +this branch. + +Valid values are a short branch name (e.g. 'master'), a full ref +name (e.g. 'refs/heads/master'), or '.' to request using the same +branch name as the manifest branch itself. Since '.' automatically +uses the manifest branch, '.' is the recommended value. + +If this key is not set, Gerrit Code Review will NOT automatically +update the gitlink. An unset key requires the manifest maintainer +to manually update the gitlink when it is necessary to reference +a different revision of the project. + +submodule..update +----------------------- + +This key is not supported by repo. If set, it will be ignored. + +.review +======= + +The optional '.review' file, located in the top-level directory of +the client's working tree (or manifest repository), is a text file +with a syntax matching the requirements of 'git config'. + +This file describes how `repo upload` should interact with the +project's preferred code review system. + +review.url +---------- + +URL of the default Gerrit Code Review server. If a project does +not have a specific URL in the '.review' file, this default URL +will be used instead. + +review..url +----------------- + +Project specific URL of the Gerrit Code Review server, for the +submodule whose project name is . + +Example +======= + + $ cat .gitmodules + [submodule "app/Clock"] + path = clock + url = git://vcs.example.com/ClockWidget.git + revision = . + [submodule "app/Browser"] + path = net/browser + url = git://netgroup.example.com/network/web/Browser.git + revision = . + + $ cat .review + [review] + url = vcs-gerrit.example.com + [review "app/Browser"] + url = netgroup.example.com + +In the above example, the app/Clock project will send its code +reviews to the default server, vcs-gerrit.example.com, while +app/Browser will send its code reviews to netgroup.example.com. + +See Also +======== + + * http://www.kernel.org/pub/software/scm/git/docs/gitmodules.html + * http://www.kernel.org/pub/software/scm/git/docs/git-config.html + * http://code.google.com/p/gerrit/ diff --git a/manifest_loader.py b/manifest_loader.py index 1ce1c1f3..467cb42a 100644 --- a/manifest_loader.py +++ b/manifest_loader.py @@ -13,11 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from manifest_submodule import SubmoduleManifest from manifest_xml import XmlManifest def ParseManifest(repodir, type=None): if type: return type(repodir) + if SubmoduleManifest.Is(repodir): + return SubmoduleManifest(repodir) return XmlManifest(repodir) _manifest = None diff --git a/manifest_submodule.py b/manifest_submodule.py new file mode 100644 index 00000000..92f187a0 --- /dev/null +++ b/manifest_submodule.py @@ -0,0 +1,474 @@ +# +# Copyright (C) 2009 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import shutil + +from error import GitError +from error import ManifestParseError +from git_command import GitCommand +from git_config import GitConfig +from git_config import IsId +from manifest import Manifest +from progress import Progress +from project import RemoteSpec +from project import Project +from project import MetaProject +from project import R_HEADS +from project import HEAD +from project import _lwrite + +import manifest_xml + +GITLINK = '160000' + +def _rmdir(dir, top): + while dir != top: + try: + os.rmdir(dir) + except OSError: + break + dir = os.path.dirname(dir) + +def _rmref(gitdir, ref): + os.remove(os.path.join(gitdir, ref)) + log = os.path.join(gitdir, 'logs', ref) + if os.path.exists(log): + os.remove(log) + _rmdir(os.path.dirname(log), gitdir) + +def _has_gitmodules(d): + return os.path.exists(os.path.join(d, '.gitmodules')) + +class SubmoduleManifest(Manifest): + """manifest from .gitmodules file""" + + @classmethod + def Is(cls, repodir): + return _has_gitmodules(os.path.dirname(repodir)) \ + or _has_gitmodules(os.path.join(repodir, 'manifest')) \ + or _has_gitmodules(os.path.join(repodir, 'manifests')) + + @classmethod + def IsBare(cls, p): + try: + p.bare_git.cat_file('-e', '%s:.gitmodules' % p.GetRevisionId()) + except GitError: + return False + return True + + def __init__(self, repodir): + Manifest.__init__(self, repodir) + + gitdir = os.path.join(repodir, 'manifest.git') + config = GitConfig.ForRepository(gitdir = gitdir) + + if config.GetBoolean('repo.mirror'): + worktree = os.path.join(repodir, 'manifest') + relpath = None + else: + worktree = self.topdir + relpath = '.' + + self.manifestProject = MetaProject(self, '__manifest__', + gitdir = gitdir, + worktree = worktree, + relpath = relpath) + self._modules = GitConfig(os.path.join(worktree, '.gitmodules'), + pickleFile = os.path.join( + repodir, '.repopickle_gitmodules' + )) + self._review = GitConfig(os.path.join(worktree, '.review'), + pickleFile = os.path.join( + repodir, '.repopickle_review' + )) + self._Unload() + + @property + def projects(self): + self._Load() + return self._projects + + def InitBranch(self): + m = self.manifestProject + if m.CurrentBranch is None: + b = m.revisionExpr + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + return m.StartBranch(b) + return True + + def SetMRefs(self, project): + if project.revisionId is None: + # Special project, e.g. the manifest or repo executable. + # + return + + ref = 'refs/remotes/m' + cur = project.bare_ref.get(ref) + exp = project.revisionId + if cur != exp: + msg = 'manifest set to %s' % exp + project.bare_git.UpdateRef(ref, exp, message = msg, detach = True) + + ref = 'refs/remotes/m-revision' + cur = project.bare_ref.symref(ref) + exp = project.revisionExpr + if exp is None: + if cur: + _rmref(project.gitdir, ref) + elif cur != exp: + remote = project.GetRemote(project.remote.name) + dst = remote.ToLocal(exp) + msg = 'manifest set to %s (%s)' % (exp, dst) + project.bare_git.symbolic_ref('-m', msg, ref, dst) + + def Upgrade_Local(self, old): + if isinstance(old, manifest_xml.XmlManifest): + self.FromXml_Local_1(old, checkout=True) + self.FromXml_Local_2(old) + else: + raise ManifestParseError, 'cannot upgrade manifest' + + def FromXml_Local_1(self, old, checkout): + os.rename(old.manifestProject.gitdir, + os.path.join(old.repodir, 'manifest.git')) + + oldmp = old.manifestProject + oldBranch = oldmp.CurrentBranch + b = oldmp.GetBranch(oldBranch).merge + if not b: + raise ManifestParseError, 'cannot upgrade manifest' + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + + newmp = self.manifestProject + self._CleanOldMRefs(newmp) + if oldBranch != b: + newmp.bare_git.branch('-m', oldBranch, b) + newmp.config.ClearCache() + + old_remote = newmp.GetBranch(b).remote.name + act_remote = self._GuessRemoteName(old) + if old_remote != act_remote: + newmp.bare_git.remote('rename', old_remote, act_remote) + newmp.config.ClearCache() + newmp.remote.name = act_remote + print >>sys.stderr, "Assuming remote named '%s'" % act_remote + + if checkout: + for p in old.projects.values(): + for c in p.copyfiles: + if os.path.exists(c.abs_dest): + os.remove(c.abs_dest) + newmp._InitWorkTree() + else: + newmp._LinkWorkTree() + + _lwrite(os.path.join(newmp.worktree,'.git',HEAD), + 'ref: refs/heads/%s\n' % b) + + def _GuessRemoteName(self, old): + used = {} + for p in old.projects.values(): + n = p.remote.name + used[n] = used.get(n, 0) + 1 + + remote_name = 'origin' + remote_used = 0 + for n in used.keys(): + if remote_used < used[n]: + remote_used = used[n] + remote_name = n + return remote_name + + def FromXml_Local_2(self, old): + shutil.rmtree(old.manifestProject.worktree) + os.remove(old._manifestFile) + + my_remote = self._Remote().name + new_base = os.path.join(self.repodir, 'projects') + old_base = os.path.join(self.repodir, 'projects.old') + os.rename(new_base, old_base) + os.makedirs(new_base) + + info = [] + pm = Progress('Converting projects', len(self.projects)) + for p in self.projects.values(): + pm.update() + + old_p = old.projects.get(p.name) + old_gitdir = os.path.join(old_base, '%s.git' % p.relpath) + if not os.path.isdir(old_gitdir): + continue + + parent = os.path.dirname(p.gitdir) + if not os.path.isdir(parent): + os.makedirs(parent) + os.rename(old_gitdir, p.gitdir) + _rmdir(os.path.dirname(old_gitdir), self.repodir) + + if not os.path.isdir(p.worktree): + os.makedirs(p.worktree) + + if os.path.isdir(os.path.join(p.worktree, '.git')): + p._LinkWorkTree(relink=True) + + self._CleanOldMRefs(p) + if old_p and old_p.remote.name != my_remote: + info.append("%s/: renamed remote '%s' to '%s'" \ + % (p.relpath, old_p.remote.name, my_remote)) + p.bare_git.remote('rename', old_p.remote.name, my_remote) + p.config.ClearCache() + + self.SetMRefs(p) + pm.end() + for i in info: + print >>sys.stderr, i + + def _CleanOldMRefs(self, p): + all_refs = p._allrefs + for ref in all_refs.keys(): + if ref.startswith(manifest_xml.R_M): + if p.bare_ref.symref(ref) != '': + _rmref(p.gitdir, ref) + else: + p.bare_git.DeleteRef(ref, all_refs[ref]) + + def FromXml_Definition(self, old): + """Convert another manifest representation to this one. + """ + mp = self.manifestProject + gm = self._modules + gr = self._review + + fd = open(os.path.join(mp.worktree, '.gitignore'), 'ab') + fd.write('/.repo\n') + fd.close() + + sort_projects = list(old.projects.keys()) + sort_projects.sort() + + b = mp.GetBranch(mp.CurrentBranch).merge + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + + info = [] + pm = Progress('Converting manifest', len(sort_projects)) + for p in sort_projects: + pm.update() + p = old.projects[p] + + gm.SetString('submodule.%s.path' % p.name, p.relpath) + gm.SetString('submodule.%s.url' % p.name, p.remote.url) + + if gr.GetString('review.url') is None: + gr.SetString('review.url', p.remote.review) + elif gr.GetString('review.url') != p.remote.review: + gr.SetString('review.%s.url' % p.name, p.remote.review) + + r = p.revisionExpr + if r and not IsId(r): + if r.startswith(R_HEADS): + r = r[len(R_HEADS):] + if r == b: + r = '.' + gm.SetString('submodule.%s.revision' % p.name, r) + + for c in p.copyfiles: + info.append('Moved %s out of %s' % (c.src, p.relpath)) + c._Copy() + p.work_git.rm(c.src) + mp.work_git.add(c.dest) + + self.SetRevisionId(p.relpath, p.GetRevisionId()) + mp.work_git.add('.gitignore', '.gitmodules', '.review') + pm.end() + for i in info: + print >>sys.stderr, i + + def _Unload(self): + self._loaded = False + self._projects = {} + self._revisionIds = None + self.branch = None + + def _Load(self): + if not self._loaded: + f = os.path.join(self.repodir, manifest_xml.LOCAL_MANIFEST_NAME) + if os.path.exists(f): + print >>sys.stderr, 'warning: ignoring %s' % f + + m = self.manifestProject + b = m.CurrentBranch + if not b: + raise ManifestParseError, 'manifest cannot be on detached HEAD' + b = m.GetBranch(b).merge + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + self.branch = b + m.remote.name = self._Remote().name + + self._ParseModules() + + if self.IsMirror: + self._AddMetaProjectMirror(self.repoProject) + self._AddMetaProjectMirror(self.manifestProject) + + self._loaded = True + + def _ParseModules(self): + byPath = dict() + for name in self._modules.GetSubSections('submodule'): + p = self._ParseProject(name) + if self._projects.get(p.name): + raise ManifestParseError, 'duplicate project "%s"' % p.name + if byPath.get(p.relpath): + raise ManifestParseError, 'duplicate path "%s"' % p.relpath + self._projects[p.name] = p + byPath[p.relpath] = p + + for relpath in self._allRevisionIds.keys(): + if relpath not in byPath: + raise ManifestParseError, \ + 'project "%s" not in .gitmodules' \ + % relpath + + def _Remote(self): + m = self.manifestProject + b = m.GetBranch(m.CurrentBranch) + return b.remote + + def _ResolveUrl(self, url): + if url.startswith('./') or url.startswith('../'): + base = self._Remote().url + try: + base = base[:base.rindex('/')+1] + except ValueError: + base = base[:base.rindex(':')+1] + if url.startswith('./'): + url = url[2:] + while '/' in base and url.startswith('../'): + base = base[:base.rindex('/')+1] + url = url[3:] + return base + url + return url + + def _GetRevisionId(self, path): + return self._allRevisionIds.get(path) + + @property + def _allRevisionIds(self): + if self._revisionIds is None: + a = dict() + p = GitCommand(self.manifestProject, + ['ls-files','-z','--stage'], + capture_stdout = True) + for line in p.process.stdout.read().split('\0')[:-1]: + l_info, l_path = line.split('\t', 2) + l_mode, l_id, l_stage = l_info.split(' ', 2) + if l_mode == GITLINK and l_stage == '0': + a[l_path] = l_id + p.Wait() + self._revisionIds = a + return self._revisionIds + + def SetRevisionId(self, path, id): + self.manifestProject.work_git.update_index( + '--add','--cacheinfo', GITLINK, id, path) + + def _ParseProject(self, name): + gm = self._modules + gr = self._review + + path = gm.GetString('submodule.%s.path' % name) + if not path: + path = name + + revId = self._GetRevisionId(path) + if not revId: + raise ManifestParseError( + 'submodule "%s" has no revision at "%s"' \ + % (name, path)) + + url = gm.GetString('submodule.%s.url' % name) + if not url: + url = name + url = self._ResolveUrl(url) + + review = gr.GetString('review.%s.url' % name) + if not review: + review = gr.GetString('review.url') + if not review: + review = self._Remote().review + + remote = RemoteSpec(self._Remote().name, url, review) + revExpr = gm.GetString('submodule.%s.revision' % name) + if revExpr == '.': + revExpr = self.branch + + if self.IsMirror: + relpath = None + worktree = None + gitdir = os.path.join(self.topdir, '%s.git' % name) + else: + worktree = os.path.join(self.topdir, path) + gitdir = os.path.join(self.repodir, 'projects/%s.git' % name) + + return Project(manifest = self, + name = name, + remote = remote, + gitdir = gitdir, + worktree = worktree, + relpath = path, + revisionExpr = revExpr, + revisionId = revId) + + def _AddMetaProjectMirror(self, m): + m_url = m.GetRemote(m.remote.name).url + if m_url.endswith('/.git'): + raise ManifestParseError, 'refusing to mirror %s' % m_url + + name = self._GuessMetaName(m_url) + if name.endswith('.git'): + name = name[:-4] + + if name not in self._projects: + m.PreSync() + gitdir = os.path.join(self.topdir, '%s.git' % name) + project = Project(manifest = self, + name = name, + remote = RemoteSpec(self._Remote().name, m_url), + gitdir = gitdir, + worktree = None, + relpath = None, + revisionExpr = m.revisionExpr, + revisionId = None) + self._projects[project.name] = project + + def _GuessMetaName(self, m_url): + parts = m_url.split('/') + name = parts[-1] + parts = parts[0:-1] + s = len(parts) - 1 + while s > 0: + l = '/'.join(parts[0:s]) + '/' + r = '/'.join(parts[s:]) + '/' + for p in self._projects.values(): + if p.name.startswith(r) and p.remote.url.startswith(l): + return r + name + s -= 1 + return m_url[m_url.rindex('/') + 1:] diff --git a/subcmds/init.py b/subcmds/init.py index b5207fbf..cdbbfdf7 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -21,6 +21,7 @@ from command import InteractiveCommand, MirrorSafeCommand from error import ManifestParseError from project import SyncBuffer from git_command import git_require, MIN_GIT_VERSION +from manifest_submodule import SubmoduleManifest from manifest_xml import XmlManifest from subcmds.sync import _ReloadManifest @@ -144,6 +145,14 @@ to update the working directory files. print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url sys.exit(1) + if is_new and SubmoduleManifest.IsBare(m): + new = self.GetManifest(reparse=True, type=SubmoduleManifest) + if m.gitdir != new.manifestProject.gitdir: + os.rename(m.gitdir, new.manifestProject.gitdir) + new = self.GetManifest(reparse=True, type=SubmoduleManifest) + m = new.manifestProject + self._ApplyOptions(opt, is_new) + if not is_new: # Force the manifest to load if it exists, the old graph # may be needed inside of _ReloadManifest(). diff --git a/subcmds/manifest.py b/subcmds/manifest.py index 551b13bd..7a8b2ee8 100644 --- a/subcmds/manifest.py +++ b/subcmds/manifest.py @@ -22,7 +22,7 @@ from manifest_xml import XmlManifest def _doc(name): r = os.path.dirname(__file__) r = os.path.dirname(r) - fd = open(os.path.join(r, 'docs', 'manifest_xml.txt')) + fd = open(os.path.join(r, 'docs', name)) try: return fd.read() finally: @@ -48,6 +48,8 @@ in a Git repository for use during future 'repo init' invocations. help = '' if isinstance(self.manifest, XmlManifest): help += self._xmlHelp + '\n' + _doc('manifest_xml.txt') + if isinstance(self.manifest, SubmoduleManifest): + help += _doc('manifest_submodule.txt') return help def _Options(self, p):