From 98ea26b8d842d11afe6326f026ba15644dc40770 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 4 Jun 2009 19:19:11 -0700 Subject: [PATCH 01/32] Allow callers to reset the git config cache If commands modify the git config too rapidly we might not notice the .git/config file has been modified, as they could run in the same filesystem timestamp window and thus not cause a change on the config's mtime. This can cause repo to miss re-reading the config file after running a command. Allowing the cache to be flushed forces us to re-read the config. Signed-off-by: Shawn O. Pearce --- git_config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/git_config.py b/git_config.py index e1e20463..2655939b 100644 --- a/git_config.py +++ b/git_config.py @@ -71,6 +71,14 @@ class GitConfig(object): else: self._pickle = pickleFile + def ClearCache(self): + if os.path.exists(self._pickle): + os.remove(self._pickle) + self._cache_dict = None + self._section_dict = None + self._remotes = {} + self._branches = {} + def Has(self, name, include_defaults = True): """Return true if this configuration file has the key. """ From ca3d8ff4fc7bac11a747e4f32a81b42a01f4f297 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 4 Jun 2009 19:49:36 -0700 Subject: [PATCH 02/32] Teach Project how to relink a .git/ in the work tree The _LinkWorkTree method can now be used to relink the work tree, such as if the real repository was moved to a different location on disk. Signed-off-by: Shawn O. Pearce --- project.py | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/project.py b/project.py index 1beee2a6..bedc91ee 100644 --- a/project.py +++ b/project.py @@ -1102,30 +1102,37 @@ class Project(object): msg = 'manifest set to %s' % self.revisionExpr self.bare_git.symbolic_ref('-m', msg, ref, dst) + def _LinkWorkTree(self, relink=False): + dotgit = os.path.join(self.worktree, '.git') + if not relink: + os.makedirs(dotgit) + + for name in ['config', + 'description', + 'hooks', + 'info', + 'logs', + 'objects', + 'packed-refs', + 'refs', + 'rr-cache', + 'svn']: + try: + src = os.path.join(self.gitdir, name) + dst = os.path.join(dotgit, name) + if relink: + os.remove(dst) + os.symlink(relpath(src, dst), dst) + except OSError, e: + if e.errno == errno.EPERM: + raise GitError('filesystem must support symlinks') + else: + raise + def _InitWorkTree(self): dotgit = os.path.join(self.worktree, '.git') if not os.path.exists(dotgit): - os.makedirs(dotgit) - - for name in ['config', - 'description', - 'hooks', - 'info', - 'logs', - 'objects', - 'packed-refs', - 'refs', - 'rr-cache', - 'svn']: - try: - src = os.path.join(self.gitdir, name) - dst = os.path.join(dotgit, name) - os.symlink(relpath(src, dst), dst) - except OSError, e: - if e.errno == errno.EPERM: - raise GitError('filesystem must support symlinks') - else: - raise + self._LinkWorkTree() _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId()) From f1a6b14fdc5402f9ed765a8a342d9c07c5b91e2d Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Wed, 3 Jun 2009 16:01:11 -0700 Subject: [PATCH 03/32] Create an abstract Manifest base class This will help as we add support for another manifest type. Signed-off-by: Shawn O. Pearce --- command.py | 10 +++++++- main.py | 4 ---- manifest.py | 37 ++++++++++++++++++++++++++++++ manifest_loader.py | 27 ++++++++++++++++++++++ manifest_xml.py | 57 +++++++++++++++++++--------------------------- subcmds/help.py | 1 + subcmds/sync.py | 2 +- 7 files changed, 99 insertions(+), 39 deletions(-) create mode 100644 manifest.py create mode 100644 manifest_loader.py diff --git a/command.py b/command.py index a941b95a..5ca43f20 100644 --- a/command.py +++ b/command.py @@ -17,6 +17,8 @@ import os import optparse import sys +import manifest_loader + from error import NoSuchProjectError class Command(object): @@ -24,7 +26,6 @@ class Command(object): """ common = False - manifest = None _optparse = None def WantPager(self, opt): @@ -57,6 +58,13 @@ class Command(object): """ raise NotImplementedError + @property + def manifest(self): + return self.GetManifest() + + def GetManifest(self, reparse=False): + return manifest_loader.GetManifest(self.repodir, reparse) + def GetProjects(self, args, missing_ok=False): """A list of projects that match the arguments. """ diff --git a/main.py b/main.py index 70ddeffa..a60641d7 100755 --- a/main.py +++ b/main.py @@ -32,11 +32,9 @@ from git_config import close_ssh from command import InteractiveCommand from command import MirrorSafeCommand from command import PagedCommand -from editor import Editor from error import ManifestInvalidRevisionError from error import NoSuchProjectError from error import RepoChangedException -from manifest_xml import XmlManifest from pager import RunPager from subcmds import all as all_commands @@ -97,8 +95,6 @@ class _Repo(object): sys.exit(1) cmd.repodir = self.repodir - cmd.manifest = XmlManifest(cmd.repodir) - Editor.globalConfig = cmd.manifest.globalConfig if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror: print >>sys.stderr, \ diff --git a/manifest.py b/manifest.py new file mode 100644 index 00000000..bf801dfa --- /dev/null +++ b/manifest.py @@ -0,0 +1,37 @@ +# +# Copyright (C) 2009 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from editor import Editor +from git_config import GitConfig +from project import MetaProject + +class Manifest(object): + """any manifest format""" + + def __init__(self, repodir): + self.repodir = os.path.abspath(repodir) + self.topdir = os.path.dirname(self.repodir) + self.globalConfig = GitConfig.ForUser() + Editor.globalConfig = self.globalConfig + + self.repoProject = MetaProject(self, 'repo', + gitdir = os.path.join(repodir, 'repo/.git'), + worktree = os.path.join(repodir, 'repo')) + + @property + def IsMirror(self): + return self.manifestProject.config.GetBoolean('repo.mirror') diff --git a/manifest_loader.py b/manifest_loader.py new file mode 100644 index 00000000..85ad3ea1 --- /dev/null +++ b/manifest_loader.py @@ -0,0 +1,27 @@ +# +# Copyright (C) 2009 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from manifest_xml import XmlManifest + +def ParseManifest(repodir): + return XmlManifest(repodir) + +_manifest = None + +def GetManifest(repodir, reparse=False): + global _manifest + if _manifest is None or reparse: + _manifest = ParseManifest(repodir) + return _manifest diff --git a/manifest_xml.py b/manifest_xml.py index 7d02f9d6..971cf212 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -18,6 +18,7 @@ import sys import xml.dom.minidom from git_config import GitConfig, IsId +from manifest import Manifest from project import RemoteSpec, Project, MetaProject, R_HEADS, HEAD from error import ManifestParseError @@ -46,19 +47,13 @@ class _XmlRemote(object): url += '/%s.git' % projectName return RemoteSpec(self.name, url, self.reviewUrl) -class XmlManifest(object): +class XmlManifest(Manifest): """manages the repo configuration file""" def __init__(self, repodir): - self.repodir = os.path.abspath(repodir) - self.topdir = os.path.dirname(self.repodir) - self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME) - self.globalConfig = GitConfig.ForUser() - - self.repoProject = MetaProject(self, 'repo', - gitdir = os.path.join(repodir, 'repo/.git'), - worktree = os.path.join(repodir, 'repo')) + Manifest.__init__(self, repodir) + self._manifestFile = os.path.join(repodir, MANIFEST_FILE_NAME) self.manifestProject = MetaProject(self, 'manifests', gitdir = os.path.join(repodir, 'manifests.git'), worktree = os.path.join(repodir, 'manifests')) @@ -72,18 +67,18 @@ class XmlManifest(object): if not os.path.isfile(path): raise ManifestParseError('manifest %s not found' % name) - old = self.manifestFile + old = self._manifestFile try: - self.manifestFile = path + self._manifestFile = path self._Unload() self._Load() finally: - self.manifestFile = old + self._manifestFile = old try: - if os.path.exists(self.manifestFile): - os.remove(self.manifestFile) - os.symlink('manifests/%s' % name, self.manifestFile) + if os.path.exists(self._manifestFile): + os.remove(self._manifestFile) + os.symlink('manifests/%s' % name, self._manifestFile) except OSError, e: raise ManifestParseError('cannot link manifest %s' % name) @@ -168,10 +163,6 @@ class XmlManifest(object): self._Load() return self._default - @property - def IsMirror(self): - return self.manifestProject.config.GetBoolean('repo.mirror') - def _Unload(self): self._loaded = False self._projects = {} @@ -192,11 +183,11 @@ class XmlManifest(object): local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME) if os.path.exists(local): try: - real = self.manifestFile - self.manifestFile = local + real = self._manifestFile + self._manifestFile = local self._ParseManifest(False) finally: - self.manifestFile = real + self._manifestFile = real if self.IsMirror: self._AddMetaProjectMirror(self.repoProject) @@ -205,17 +196,17 @@ class XmlManifest(object): self._loaded = True def _ParseManifest(self, is_root_file): - root = xml.dom.minidom.parse(self.manifestFile) + root = xml.dom.minidom.parse(self._manifestFile) if not root or not root.childNodes: raise ManifestParseError, \ "no root node in %s" % \ - self.manifestFile + self._manifestFile config = root.childNodes[0] if config.nodeName != 'manifest': raise ManifestParseError, \ "no in %s" % \ - self.manifestFile + self._manifestFile for node in config.childNodes: if node.nodeName == 'remove-project': @@ -233,7 +224,7 @@ class XmlManifest(object): if self._remotes.get(remote.name): raise ManifestParseError, \ 'duplicate remote %s in %s' % \ - (remote.name, self.manifestFile) + (remote.name, self._manifestFile) self._remotes[remote.name] = remote for node in config.childNodes: @@ -241,7 +232,7 @@ class XmlManifest(object): if self._default is not None: raise ManifestParseError, \ 'duplicate default in %s' % \ - (self.manifestFile) + (self._manifestFile) self._default = self._ParseDefault(node) if self._default is None: self._default = _Default() @@ -252,7 +243,7 @@ class XmlManifest(object): if self._projects.get(project.name): raise ManifestParseError, \ 'duplicate project %s in %s' % \ - (project.name, self.manifestFile) + (project.name, self._manifestFile) self._projects[project.name] = project def _AddMetaProjectMirror(self, m): @@ -324,7 +315,7 @@ class XmlManifest(object): if remote is None: raise ManifestParseError, \ "no remote for project %s within %s" % \ - (name, self.manifestFile) + (name, self._manifestFile) revisionExpr = node.getAttribute('revision') if not revisionExpr: @@ -332,7 +323,7 @@ class XmlManifest(object): if not revisionExpr: raise ManifestParseError, \ "no revision for project %s within %s" % \ - (name, self.manifestFile) + (name, self._manifestFile) path = node.getAttribute('path') if not path: @@ -340,7 +331,7 @@ class XmlManifest(object): if path.startswith('/'): raise ManifestParseError, \ "project %s path cannot be absolute in %s" % \ - (name, self.manifestFile) + (name, self._manifestFile) if self.IsMirror: relpath = None @@ -382,7 +373,7 @@ class XmlManifest(object): if not v: raise ManifestParseError, \ "remote %s not defined in %s" % \ - (name, self.manifestFile) + (name, self._manifestFile) return v def _reqatt(self, node, attname): @@ -393,5 +384,5 @@ class XmlManifest(object): if not v: raise ManifestParseError, \ "no %s in <%s> within %s" % \ - (attname, node.nodeName, self.manifestFile) + (attname, node.nodeName, self._manifestFile) return v diff --git a/subcmds/help.py b/subcmds/help.py index c5979fd6..01d5fa23 100644 --- a/subcmds/help.py +++ b/subcmds/help.py @@ -163,6 +163,7 @@ See 'repo help --all' for a complete list of recognized commands. print >>sys.stderr, "repo: '%s' is not a repo command." % name sys.exit(1) + cmd.repodir = self.repodir self._PrintCommandHelp(cmd) else: diff --git a/subcmds/sync.py b/subcmds/sync.py index bd07dd9f..afd44dab 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -214,7 +214,7 @@ uncommitted changes are present' % project.relpath if not syncbuf.Finish(): sys.exit(1) - self.manifest._Unload() + self.GetManifest(reparse=True) all = self.GetProjects(args, missing_ok=True) missing = [] for project in all: From 60e679209a5495393ef584efaaad287fc8b77c51 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Wed, 3 Jun 2009 17:43:16 -0700 Subject: [PATCH 04/32] help: Don't show empty Summary or Description sections Signed-off-by: Shawn O. Pearce --- subcmds/help.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subcmds/help.py b/subcmds/help.py index 01d5fa23..e2f3074c 100644 --- a/subcmds/help.py +++ b/subcmds/help.py @@ -94,6 +94,8 @@ See 'repo help --all' for a complete list of recognized commands. body = getattr(cmd, bodyAttr) except AttributeError: return + if body == '' or body is None: + return self.nl() From 050e4fd591537811e6e62b2e9ba1ce83e520e550 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Wed, 3 Jun 2009 14:16:14 -0700 Subject: [PATCH 05/32] manifest: Only display XML help on XML manifest Some of the help text is only related to the XML formatted manifest, so only display that text if that is the current format. Signed-off-by: Shawn O. Pearce --- .../{manifest-format.txt => manifest_xml.txt} | 0 subcmds/manifest.py | 21 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) rename docs/{manifest-format.txt => manifest_xml.txt} (100%) diff --git a/docs/manifest-format.txt b/docs/manifest_xml.txt similarity index 100% rename from docs/manifest-format.txt rename to docs/manifest_xml.txt diff --git a/subcmds/manifest.py b/subcmds/manifest.py index 4374a9d0..4415b99e 100644 --- a/subcmds/manifest.py +++ b/subcmds/manifest.py @@ -18,13 +18,22 @@ import sys from command import PagedCommand +def _doc(name): + r = os.path.dirname(__file__) + r = os.path.dirname(r) + fd = open(os.path.join(r, 'docs', 'manifest_xml.txt')) + try: + return fd.read() + finally: + fd.close() + class Manifest(PagedCommand): common = False helpSummary = "Manifest inspection utility" helpUsage = """ %prog [-o {-|NAME.xml} [-r]] """ - _helpDescription = """ + _xmlHelp = """ With the -o option, exports the current manifest for inspection. The manifest and (if present) local_manifest.xml are combined @@ -35,13 +44,9 @@ in a Git repository for use during future 'repo init' invocations. @property def helpDescription(self): - help = self._helpDescription + '\n' - r = os.path.dirname(__file__) - r = os.path.dirname(r) - fd = open(os.path.join(r, 'docs', 'manifest-format.txt')) - for line in fd: - help += line - fd.close() + help = '' + if isinstance(self.manifest, XmlManifest): + help += self._xmlHelp + '\n' + _doc('manifest_xml.txt') return help def _Options(self, p): From 67f4563acb58af9e64fbfe8a2c9794b389a3debc Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 19 May 2009 18:17:51 -0700 Subject: [PATCH 06/32] manifest: Only support -o option on XML formatted manifest If the manifest isn't a single file format manifest, the -o option makes no sense, as you cannot export multiple files to a single stream for display or redirection. Signed-off-by: Shawn O. Pearce --- subcmds/manifest.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/subcmds/manifest.py b/subcmds/manifest.py index 4415b99e..551b13bd 100644 --- a/subcmds/manifest.py +++ b/subcmds/manifest.py @@ -17,6 +17,7 @@ import os import sys from command import PagedCommand +from manifest_xml import XmlManifest def _doc(name): r = os.path.dirname(__file__) @@ -31,7 +32,7 @@ class Manifest(PagedCommand): common = False helpSummary = "Manifest inspection utility" helpUsage = """ -%prog [-o {-|NAME.xml} [-r]] +%prog [options] """ _xmlHelp = """ @@ -50,13 +51,14 @@ in a Git repository for use during future 'repo init' invocations. return help def _Options(self, p): - p.add_option('-r', '--revision-as-HEAD', - dest='peg_rev', action='store_true', - help='Save revisions as current HEAD') - p.add_option('-o', '--output-file', - dest='output_file', - help='File to save the manifest to', - metavar='-|NAME.xml') + if isinstance(self.manifest, XmlManifest): + p.add_option('-r', '--revision-as-HEAD', + dest='peg_rev', action='store_true', + help='Save revisions as current HEAD') + p.add_option('-o', '--output-file', + dest='output_file', + help='File to save the manifest to', + metavar='-|NAME.xml') def _Output(self, opt): if opt.output_file == '-': @@ -73,9 +75,10 @@ in a Git repository for use during future 'repo init' invocations. if args: self.Usage() - if opt.output_file is not None: - self._Output(opt) - return + if isinstance(self.manifest, XmlManifest) \ + and opt.output_file is not None: + self._Output(opt) + return print >>sys.stderr, 'error: no operation to perform' print >>sys.stderr, 'error: see repo help manifest' From 446c4e5556a4c85621d61b3aba63d084300c6224 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 19 May 2009 18:14:04 -0700 Subject: [PATCH 07/32] init: Allow -m only on XML formatted manifest If the manifest is the newer SubmoduleManifest style, then the -m option makes no sense, as you cannot select a specific file within the current branch. Signed-off-by: Shawn O. Pearce --- subcmds/init.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/subcmds/init.py b/subcmds/init.py index 75a58f11..ec87d036 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -21,6 +21,7 @@ from command import InteractiveCommand, MirrorSafeCommand from error import ManifestParseError from project import SyncBuffer from git_command import git_require, MIN_GIT_VERSION +from manifest_xml import XmlManifest class Init(InteractiveCommand, MirrorSafeCommand): common = True @@ -37,10 +38,6 @@ current working directory. The optional -b argument can be used to select the manifest branch to checkout and use. If no branch is specified, master is assumed. -The optional -m argument can be used to specify an alternate manifest -to be used. If no manifest is specified, the manifest default.xml -will be used. - Switching Manifest Branches --------------------------- @@ -65,9 +62,11 @@ to update the working directory files. g.add_option('-b', '--manifest-branch', dest='manifest_branch', help='manifest branch or revision', metavar='REVISION') - g.add_option('-m', '--manifest-name', - dest='manifest_name', default='default.xml', - help='initial manifest file', metavar='NAME.xml') + if isinstance(self.manifest, XmlManifest) \ + or not self.manifest.manifestProject.Exists: + g.add_option('-m', '--manifest-name', + dest='manifest_name', default='default.xml', + help='initial manifest file', metavar='NAME.xml') g.add_option('--mirror', dest='mirror', action='store_true', help='mirror the forrest') @@ -216,7 +215,8 @@ to update the working directory files. def Execute(self, opt, args): git_require(MIN_GIT_VERSION, fail=True) self._SyncManifest(opt) - self._LinkManifest(opt.manifest_name) + if isinstance(self.manifest, XmlManifest): + self._LinkManifest(opt.manifest_name) if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror: self._ConfigureUser() From 1875ddd47c0bf38e5cc52e1e5875caabce2d8742 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 15:22:49 -0700 Subject: [PATCH 08/32] sync: Run `git gc --auto` after fetch Users may wind up with a lot of loose object content in projects they don't frequently make changes in, but that are modified by others. Since we bypass many git code paths that would have otherwise called out to `git gc --auto`, its possible for these projects to have their loose object database grow out of control. To help prevent that, we now invoke it ourselves during the network half of sync. Signed-off-by: Shawn O. Pearce --- subcmds/selfupdate.py | 1 + subcmds/sync.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/subcmds/selfupdate.py b/subcmds/selfupdate.py index 4f46a129..46aa3a19 100644 --- a/subcmds/selfupdate.py +++ b/subcmds/selfupdate.py @@ -55,6 +55,7 @@ need to be performed by an end-user. print >>sys.stderr, "error: can't update repo" sys.exit(1) + rp.bare_git.gc('--auto') _PostRepoFetch(rp, no_repo_verify = opt.no_repo_verify, verbose = True) diff --git a/subcmds/sync.py b/subcmds/sync.py index afd44dab..1537c9a2 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -118,6 +118,8 @@ later is required to fix a server side protocol bug. print >>sys.stderr, 'error: Cannot fetch %s' % project.name sys.exit(1) pm.end() + for project in projects: + project.bare_git.gc('--auto') return fetched def UpdateProjectList(self): From b0ca41e19ad0631d82194405d992a7a3b4a834fc Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 20:01:47 -0700 Subject: [PATCH 09/32] Only remove a stale pickle file if it exists Signed-off-by: Shawn O. Pearce --- git_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git_config.py b/git_config.py index 2655939b..b47bc66f 100644 --- a/git_config.py +++ b/git_config.py @@ -262,9 +262,11 @@ class GitConfig(object): finally: fd.close() except IOError: - os.remove(self._pickle) + if os.path.exists(self._pickle): + os.remove(self._pickle) except cPickle.PickleError: - os.remove(self._pickle) + if os.path.exists(self._pickle): + os.remove(self._pickle) def _ReadGit(self): """ From 2095179beec754d2d5bfe175215e736b7ff838e9 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 15:26:17 -0700 Subject: [PATCH 10/32] Cleanup import formatting in manifest_xml.py Signed-off-by: Shawn O. Pearce --- manifest_xml.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/manifest_xml.py b/manifest_xml.py index 971cf212..97df75bd 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -17,9 +17,14 @@ import os import sys import xml.dom.minidom -from git_config import GitConfig, IsId +from git_config import GitConfig +from git_config import IsId from manifest import Manifest -from project import RemoteSpec, Project, MetaProject, R_HEADS, HEAD +from project import RemoteSpec +from project import Project +from project import MetaProject +from project import R_HEADS +from project import HEAD from error import ManifestParseError MANIFEST_FILE_NAME = 'manifest.xml' From cc6c79643e1cafad565424caabe581e7b548bf6f Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 15:29:02 -0700 Subject: [PATCH 11/32] Make refs/remotes/m management the manifest object's responsibility I plan to have the new submodule manifest format use a different layout for the m refs than the XML manifest format has used in the past. Thus we need to move the behavior management into the manifest object, and out of the project, so we can change it. Signed-off-by: Shawn O. Pearce --- git_refs.py | 1 - manifest.py | 7 +++++++ manifest_xml.py | 5 +++++ project.py | 8 ++------ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/git_refs.py b/git_refs.py index ac8ed0c1..b24a0b4e 100644 --- a/git_refs.py +++ b/git_refs.py @@ -21,7 +21,6 @@ HEAD = 'HEAD' R_HEADS = 'refs/heads/' R_TAGS = 'refs/tags/' R_PUB = 'refs/published/' -R_M = 'refs/remotes/m/' class GitRefs(object): diff --git a/manifest.py b/manifest.py index bf801dfa..0762098b 100644 --- a/manifest.py +++ b/manifest.py @@ -35,3 +35,10 @@ class Manifest(object): @property def IsMirror(self): return self.manifestProject.config.GetBoolean('repo.mirror') + + @property + def projects(self): + return {} + + def SetMRefs(self, project): + pass diff --git a/manifest_xml.py b/manifest_xml.py index 97df75bd..66cdf3e3 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -29,6 +29,7 @@ from error import ManifestParseError MANIFEST_FILE_NAME = 'manifest.xml' LOCAL_MANIFEST_NAME = 'local_manifest.xml' +R_M = 'refs/remotes/m/' class _Default(object): """Project defaults within the manifest.""" @@ -168,6 +169,10 @@ class XmlManifest(Manifest): self._Load() return self._default + def SetMRefs(self, project): + if self.branch: + project._InitAnyMRef(R_M + self.branch) + def _Unload(self): self._loaded = False self._projects = {} diff --git a/project.py b/project.py index bedc91ee..6188ca72 100644 --- a/project.py +++ b/project.py @@ -27,7 +27,7 @@ from git_config import GitConfig, IsId from error import GitError, ImportError, UploadError from error import ManifestInvalidRevisionError -from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M +from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB def _lwrite(path, content): lock = '%s.lock' % path @@ -598,7 +598,7 @@ class Project(object): return False if self.worktree: - self._InitMRef() + self.manifest.SetMRefs(self) else: self._InitMirrorHead() try: @@ -1080,10 +1080,6 @@ class Project(object): remote.ResetFetch(mirror=True) remote.Save() - def _InitMRef(self): - if self.manifest.branch: - self._InitAnyMRef(R_M + self.manifest.branch) - def _InitMirrorHead(self): self._InitAnyMRef(HEAD) From abb7a3dfecdfe98b30594219f24c5c3d5e11e990 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 16:16:18 -0700 Subject: [PATCH 12/32] Allow callers to request a specific type of manifest If the caller knows exactly what the manifest type must be we can now ask the loader to directly construct that type, rather than guessing it from the working directory. Signed-off-by: Shawn O. Pearce --- command.py | 6 ++++-- manifest_loader.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/command.py b/command.py index 5ca43f20..9e970e80 100644 --- a/command.py +++ b/command.py @@ -62,8 +62,10 @@ class Command(object): def manifest(self): return self.GetManifest() - def GetManifest(self, reparse=False): - return manifest_loader.GetManifest(self.repodir, reparse) + def GetManifest(self, reparse=False, type=None): + return manifest_loader.GetManifest(self.repodir, + reparse=reparse, + type=type) def GetProjects(self, args, missing_ok=False): """A list of projects that match the arguments. diff --git a/manifest_loader.py b/manifest_loader.py index 85ad3ea1..1ce1c1f3 100644 --- a/manifest_loader.py +++ b/manifest_loader.py @@ -15,13 +15,17 @@ from manifest_xml import XmlManifest -def ParseManifest(repodir): +def ParseManifest(repodir, type=None): + if type: + return type(repodir) return XmlManifest(repodir) _manifest = None -def GetManifest(repodir, reparse=False): +def GetManifest(repodir, reparse=False, type=None): global _manifest - if _manifest is None or reparse: - _manifest = ParseManifest(repodir) + if _manifest is None \ + or reparse \ + or (type and _manifest.__class__ != type): + _manifest = ParseManifest(repodir, type=type) return _manifest From 75b87c8a5171b26947d0a13d970f09defac736e3 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 16:24:57 -0700 Subject: [PATCH 13/32] Abstract manifest branch creation from init to the manifest object This permits the XML style manifest to use 'default', while other types can use their own creation strategy for the current branch. Signed-off-by: Shawn O. Pearce --- manifest.py | 3 +++ manifest_xml.py | 6 ++++++ subcmds/init.py | 7 +++---- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/manifest.py b/manifest.py index 0762098b..a2fc9601 100644 --- a/manifest.py +++ b/manifest.py @@ -40,5 +40,8 @@ class Manifest(object): def projects(self): return {} + def InitBranch(self): + pass + def SetMRefs(self, project): pass diff --git a/manifest_xml.py b/manifest_xml.py index 66cdf3e3..45896be9 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -169,6 +169,12 @@ class XmlManifest(Manifest): self._Load() return self._default + def InitBranch(self): + m = self.manifestProject + if m.CurrentBranch is None: + return m.StartBranch('default') + return True + def SetMRefs(self, project): if self.branch: project._InitAnyMRef(R_M + self.branch) diff --git a/subcmds/init.py b/subcmds/init.py index ec87d036..0075b0b4 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -130,10 +130,9 @@ to update the working directory files. m.Sync_LocalHalf(syncbuf) syncbuf.Finish() - if is_new or m.CurrentBranch is None: - if not m.StartBranch('default'): - print >>sys.stderr, 'fatal: cannot create default in manifest' - sys.exit(1) + if not self.manifest.InitBranch(): + print >>sys.stderr, 'fatal: cannot create branch in manifest' + sys.exit(1) def _LinkManifest(self, name): if not name: From ce86abbe8ab9389fbad9d375e3754ed054d8b744 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 16:52:28 -0700 Subject: [PATCH 14/32] Allow the manifest to be accessed it if is in work tree If the manifest's work tree is actually inside of the rest of the client work tree then its only fair that we include it as a project that the user can access. Signed-off-by: Shawn O. Pearce --- command.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/command.py b/command.py index 9e970e80..6e4e2c57 100644 --- a/command.py +++ b/command.py @@ -71,6 +71,12 @@ class Command(object): """A list of projects that match the arguments. """ all = self.manifest.projects + + mp = self.manifest.manifestProject + if mp.relpath == '.': + all = dict(all) + all[mp.name] = mp + result = [] if not args: @@ -91,7 +97,9 @@ class Command(object): for p in all.values(): by_path[p.worktree] = p - if os.path.exists(path): + try: + project = by_path[path] + except KeyError: while path \ and path != '/' \ and path != self.manifest.topdir: @@ -100,11 +108,6 @@ class Command(object): break except KeyError: path = os.path.dirname(path) - else: - try: - project = by_path[path] - except KeyError: - pass if not project: raise NoSuchProjectError(arg) From 7354d8891480f5eaa535acc38a24f42ea63b18a6 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 20:06:13 -0700 Subject: [PATCH 15/32] init: Ensure repo.mirror is noticed once set If we don't clear the cache, there can be a timestamp race between the pickle file and the raw text file, and we may not pick up the edit when we create a new config object around the same path name. Signed-off-by: Shawn O. Pearce --- subcmds/init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/subcmds/init.py b/subcmds/init.py index 0075b0b4..0586721f 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -117,6 +117,7 @@ to update the working directory files. if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') + m.config.ClearCache() else: print >>sys.stderr, 'fatal: --mirror not supported on existing client' sys.exit(1) From b3d2c9214be60f575d64b3af3b87a3632de04ba0 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 17:24:45 -0700 Subject: [PATCH 16/32] init (wrapper): Note that -m is now deprecated If the manifest format isn't XML, this option isn't available. Signed-off-by: Shawn O. Pearce --- repo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repo b/repo index 13742559..f73fa58b 100755 --- a/repo +++ b/repo @@ -114,7 +114,8 @@ group.add_option('-b', '--manifest-branch', help='manifest branch or revision', metavar='REVISION') group.add_option('-m', '--manifest-name', dest='manifest_name', - help='initial manifest file', metavar='NAME.xml') + help='initial manifest file (deprecated)', + metavar='NAME.xml') group.add_option('--mirror', dest='mirror', action='store_true', help='mirror the forrest') From 5f947bba69de81f58f1adef10225c04727fa0ed5 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 17:24:17 -0700 Subject: [PATCH 17/32] init: add -o, --origin to name manifest remote The -o option permits the user to control the name of the manifest's remote, which normally is hardcoded to be 'origin', but can differ because we derive it at runtime from the configuration file. Signed-off-by: Shawn O. Pearce --- manifest_xml.py | 5 ++++- project.py | 8 +++++--- repo | 6 +++++- subcmds/init.py | 36 ++++++++++++++++++++++++++---------- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/manifest_xml.py b/manifest_xml.py index 45896be9..d888653d 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -189,7 +189,10 @@ class XmlManifest(Manifest): def _Load(self): if not self._loaded: m = self.manifestProject - b = m.GetBranch(m.CurrentBranch).merge + b = m.GetBranch(m.CurrentBranch) + if b.remote and b.remote.name: + m.remote.name = b.remote.name + b = b.merge if b is not None and b.startswith(R_HEADS): b = b[len(R_HEADS):] self.branch = b diff --git a/project.py b/project.py index 6188ca72..c79e8fb9 100644 --- a/project.py +++ b/project.py @@ -1442,10 +1442,12 @@ class MetaProject(Project): if self.Exists: cb = self.CurrentBranch if cb: - base = self.GetBranch(cb).merge - if base: - self.revisionExpr = base + cb = self.GetBranch(cb) + if cb.merge: + self.revisionExpr = cb.merge self.revisionId = None + if cb.remote and cb.remote.name: + self.remote.name = cb.remote.name @property def LastFetch(self): diff --git a/repo b/repo index f73fa58b..ff7c4188 100755 --- a/repo +++ b/repo @@ -28,7 +28,7 @@ if __name__ == '__main__': del magic # increment this whenever we make important changes to this script -VERSION = (1, 8) +VERSION = (1, 9) # increment this if the MAINTAINER_KEYS block is modified KEYRING_VERSION = (1,0) @@ -109,6 +109,10 @@ group = init_optparse.add_option_group('Manifest options') group.add_option('-u', '--manifest-url', dest='manifest_url', help='manifest repository location', metavar='URL') +group.add_option('-o', '--origin', + dest='manifest_origin', + help="use REMOTE instead of 'origin' to track upstream", + metavar='REMOTE') group.add_option('-b', '--manifest-branch', dest='manifest_branch', help='manifest branch or revision', metavar='REVISION') diff --git a/subcmds/init.py b/subcmds/init.py index 0586721f..53c3a010 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -62,6 +62,10 @@ to update the working directory files. g.add_option('-b', '--manifest-branch', dest='manifest_branch', help='manifest branch or revision', metavar='REVISION') + g.add_option('-o', '--origin', + dest='manifest_origin', + help="use REMOTE instead of 'origin' to track upstream", + metavar='REMOTE') if isinstance(self.manifest, XmlManifest) \ or not self.manifest.manifestProject.Exists: g.add_option('-m', '--manifest-name', @@ -84,6 +88,27 @@ to update the working directory files. dest='no_repo_verify', action='store_true', help='do not verify repo source code') + def _ApplyOptions(self, opt, is_new): + m = self.manifest.manifestProject + + if is_new: + if opt.manifest_origin: + m.remote.name = opt.manifest_origin + + if opt.manifest_branch: + m.revisionExpr = opt.manifest_branch + else: + m.revisionExpr = 'refs/heads/master' + else: + if opt.manifest_origin: + print >>sys.stderr, 'fatal: cannot change origin name' + sys.exit(1) + + if opt.manifest_branch: + m.revisionExpr = opt.manifest_branch + else: + m.PreSync() + def _SyncManifest(self, opt): m = self.manifest.manifestProject is_new = not m.Exists @@ -98,16 +123,7 @@ to update the working directory files. print >>sys.stderr, ' from %s' % opt.manifest_url m._InitGitDir() - if opt.manifest_branch: - m.revisionExpr = opt.manifest_branch - else: - m.revisionExpr = 'refs/heads/master' - else: - if opt.manifest_branch: - m.revisionExpr = opt.manifest_branch - else: - m.PreSync() - + self._ApplyOptions(opt, is_new) if opt.manifest_url: r = m.GetRemote(m.remote.name) r.url = opt.manifest_url From 87bda12e85ffb98778d7ac881edb0210b74c0491 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 16:37:30 -0700 Subject: [PATCH 18/32] sync: Support upgrading manifest formats If the manifest format changes during init or sync we need to do a full reparse of the manifest, and possibly allow the new object to reconfigure the local workspace to match its expectations. Signed-off-by: Shawn O. Pearce --- manifest.py | 4 ++++ subcmds/init.py | 9 +++++++++ subcmds/sync.py | 10 +++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/manifest.py b/manifest.py index a2fc9601..f737e866 100644 --- a/manifest.py +++ b/manifest.py @@ -15,6 +15,7 @@ import os +from error import ManifestParseError from editor import Editor from git_config import GitConfig from project import MetaProject @@ -45,3 +46,6 @@ class Manifest(object): def SetMRefs(self, project): pass + + def Upgrade_Local(self, old): + raise ManifestParseError, 'unsupported upgrade path' diff --git a/subcmds/init.py b/subcmds/init.py index 53c3a010..b5207fbf 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -22,6 +22,7 @@ from error import ManifestParseError from project import SyncBuffer from git_command import git_require, MIN_GIT_VERSION from manifest_xml import XmlManifest +from subcmds.sync import _ReloadManifest class Init(InteractiveCommand, MirrorSafeCommand): common = True @@ -143,9 +144,17 @@ to update the working directory files. print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url sys.exit(1) + if not is_new: + # Force the manifest to load if it exists, the old graph + # may be needed inside of _ReloadManifest(). + # + self.manifest.projects + syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf) syncbuf.Finish() + _ReloadManifest(self) + self._ApplyOptions(opt, is_new) if not self.manifest.InitBranch(): print >>sys.stderr, 'fatal: cannot create branch in manifest' diff --git a/subcmds/sync.py b/subcmds/sync.py index 1537c9a2..5fc834d0 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -215,8 +215,9 @@ uncommitted changes are present' % project.relpath mp.Sync_LocalHalf(syncbuf) if not syncbuf.Finish(): sys.exit(1) + _ReloadManifest(self) + mp = self.manifest.manifestProject - self.GetManifest(reparse=True) all = self.GetProjects(args, missing_ok=True) missing = [] for project in all: @@ -243,6 +244,13 @@ uncommitted changes are present' % project.relpath if not syncbuf.Finish(): sys.exit(1) +def _ReloadManifest(cmd): + old = cmd.manifest + new = cmd.GetManifest(reparse=True) + + if old.__class__ != new.__class__: + print >>sys.stderr, 'NOTICE: manifest format has changed ***' + new.Upgrade_Local(old) def _PostRepoUpgrade(manifest): for project in manifest.projects.values(): From a7ce096047a7707edc572de375b700d161b9520b Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 18:04:27 -0700 Subject: [PATCH 19/32] Allow meta projects to be created not under .repo/ Some types of manifests might prefer to put their meta project work tree under topdir, rather than inside of the .repo/ directory. We can support that by allowing relpath to be optionally passed in. Signed-off-by: Shawn O. Pearce --- project.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/project.py b/project.py index c79e8fb9..beacc92f 100644 --- a/project.py +++ b/project.py @@ -1426,15 +1426,17 @@ class SyncBuffer(object): class MetaProject(Project): """A special project housed under .repo. """ - def __init__(self, manifest, name, gitdir, worktree): + def __init__(self, manifest, name, gitdir, worktree, relpath=None): repodir = manifest.repodir + if relpath is None: + relpath = '.repo/%s' % name Project.__init__(self, manifest = manifest, name = name, gitdir = gitdir, worktree = worktree, remote = RemoteSpec('origin'), - relpath = '.repo/%s' % name, + relpath = relpath, revisionExpr = 'refs/heads/master', revisionId = None) From 0125ae2fda18deee89dc94b32a2daa1b37a8a361 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 18:05:23 -0700 Subject: [PATCH 20/32] Introduce manifest format using git submodules If a manifest top level directory contains '.gitmodules' we now assume this is a git module format manifest and switch to using that code, rather than the legacy XML based manifest. At the same time, we move the bare repository for a project from $TOP/.repo/projects/$REPO_PATH.git to be $REPO_NAME.git instead. This makes it easier for us to later support a repo init from an existing work tree, as we can more accurately predict the path of the project's repository in the workspace. It also means that the $TOP/.repo/projects/ directory is layed out like a mirror would be. Signed-off-by: Shawn O. Pearce --- docs/manifest_submodule.txt | 130 ++++++++++ manifest_loader.py | 3 + manifest_submodule.py | 474 ++++++++++++++++++++++++++++++++++++ subcmds/init.py | 9 + subcmds/manifest.py | 4 +- 5 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 docs/manifest_submodule.txt create mode 100644 manifest_submodule.py diff --git a/docs/manifest_submodule.txt b/docs/manifest_submodule.txt new file mode 100644 index 00000000..e7d1f643 --- /dev/null +++ b/docs/manifest_submodule.txt @@ -0,0 +1,130 @@ +repo Manifest Format (submodule) +================================ + +A repo manifest describes the structure of a repo client; that is +the directories that are visible and where they should be obtained +from with git. + +The basic structure of a manifest is a bare Git repository holding +a 'gitmodules' file in the top level directory, and one or more +gitlink references pointing at commits from the referenced projects. +This is the same structure as used by 'git submodule'. + +Manifests are inherently version controlled, since they are kept +within a Git repository. Updates to manifests are automatically +obtained by clients during `repo sync`. + +.gitmodules +=========== + +The '.gitmodules' file, located in the top-level directory of the +client's working tree (or manifest repository), is a text file with +a syntax matching the requirements of 'git config'. + +This file contains one subsection per project (also called a +submodule by git), and the subsection value is a unique name to +describe the project. Each submodule section must contain the +following required keys: + + * path + * url + +submodule..path +--------------------- + +Defines the path, relative to the top-level directory of the client's +working tree, where the project is expected to be checked out. The +path name must not end with a '/'. All paths must be unique within +the .gitmodules file. + +At the specified path within the manifest repository a gitlink +tree entry (an entry with file mode 160000) must exist referencing +a commit SHA-1 from the project. This tree entry specifies the +exact version of the project that `repo sync` will synchronize the +client's working tree to. + +submodule..url +-------------------- + +Defines a URL from where the project repository can be cloned. +By default `repo sync` will clone from this URL whenever a user +needs to access this project. + +submodule..revision +------------------------- + +Name of the branch in the project repository that Gerrit Code Review +should automatically refresh the project's gitlink entry from. + +If set, during submit of a change within the referenced project, +Gerrit Code Review will automatically update the manifest +repository's corresponding gitlink to the new commit SHA-1 of +this branch. + +Valid values are a short branch name (e.g. 'master'), a full ref +name (e.g. 'refs/heads/master'), or '.' to request using the same +branch name as the manifest branch itself. Since '.' automatically +uses the manifest branch, '.' is the recommended value. + +If this key is not set, Gerrit Code Review will NOT automatically +update the gitlink. An unset key requires the manifest maintainer +to manually update the gitlink when it is necessary to reference +a different revision of the project. + +submodule..update +----------------------- + +This key is not supported by repo. If set, it will be ignored. + +.review +======= + +The optional '.review' file, located in the top-level directory of +the client's working tree (or manifest repository), is a text file +with a syntax matching the requirements of 'git config'. + +This file describes how `repo upload` should interact with the +project's preferred code review system. + +review.url +---------- + +URL of the default Gerrit Code Review server. If a project does +not have a specific URL in the '.review' file, this default URL +will be used instead. + +review..url +----------------- + +Project specific URL of the Gerrit Code Review server, for the +submodule whose project name is . + +Example +======= + + $ cat .gitmodules + [submodule "app/Clock"] + path = clock + url = git://vcs.example.com/ClockWidget.git + revision = . + [submodule "app/Browser"] + path = net/browser + url = git://netgroup.example.com/network/web/Browser.git + revision = . + + $ cat .review + [review] + url = vcs-gerrit.example.com + [review "app/Browser"] + url = netgroup.example.com + +In the above example, the app/Clock project will send its code +reviews to the default server, vcs-gerrit.example.com, while +app/Browser will send its code reviews to netgroup.example.com. + +See Also +======== + + * http://www.kernel.org/pub/software/scm/git/docs/gitmodules.html + * http://www.kernel.org/pub/software/scm/git/docs/git-config.html + * http://code.google.com/p/gerrit/ diff --git a/manifest_loader.py b/manifest_loader.py index 1ce1c1f3..467cb42a 100644 --- a/manifest_loader.py +++ b/manifest_loader.py @@ -13,11 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from manifest_submodule import SubmoduleManifest from manifest_xml import XmlManifest def ParseManifest(repodir, type=None): if type: return type(repodir) + if SubmoduleManifest.Is(repodir): + return SubmoduleManifest(repodir) return XmlManifest(repodir) _manifest = None diff --git a/manifest_submodule.py b/manifest_submodule.py new file mode 100644 index 00000000..92f187a0 --- /dev/null +++ b/manifest_submodule.py @@ -0,0 +1,474 @@ +# +# Copyright (C) 2009 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import shutil + +from error import GitError +from error import ManifestParseError +from git_command import GitCommand +from git_config import GitConfig +from git_config import IsId +from manifest import Manifest +from progress import Progress +from project import RemoteSpec +from project import Project +from project import MetaProject +from project import R_HEADS +from project import HEAD +from project import _lwrite + +import manifest_xml + +GITLINK = '160000' + +def _rmdir(dir, top): + while dir != top: + try: + os.rmdir(dir) + except OSError: + break + dir = os.path.dirname(dir) + +def _rmref(gitdir, ref): + os.remove(os.path.join(gitdir, ref)) + log = os.path.join(gitdir, 'logs', ref) + if os.path.exists(log): + os.remove(log) + _rmdir(os.path.dirname(log), gitdir) + +def _has_gitmodules(d): + return os.path.exists(os.path.join(d, '.gitmodules')) + +class SubmoduleManifest(Manifest): + """manifest from .gitmodules file""" + + @classmethod + def Is(cls, repodir): + return _has_gitmodules(os.path.dirname(repodir)) \ + or _has_gitmodules(os.path.join(repodir, 'manifest')) \ + or _has_gitmodules(os.path.join(repodir, 'manifests')) + + @classmethod + def IsBare(cls, p): + try: + p.bare_git.cat_file('-e', '%s:.gitmodules' % p.GetRevisionId()) + except GitError: + return False + return True + + def __init__(self, repodir): + Manifest.__init__(self, repodir) + + gitdir = os.path.join(repodir, 'manifest.git') + config = GitConfig.ForRepository(gitdir = gitdir) + + if config.GetBoolean('repo.mirror'): + worktree = os.path.join(repodir, 'manifest') + relpath = None + else: + worktree = self.topdir + relpath = '.' + + self.manifestProject = MetaProject(self, '__manifest__', + gitdir = gitdir, + worktree = worktree, + relpath = relpath) + self._modules = GitConfig(os.path.join(worktree, '.gitmodules'), + pickleFile = os.path.join( + repodir, '.repopickle_gitmodules' + )) + self._review = GitConfig(os.path.join(worktree, '.review'), + pickleFile = os.path.join( + repodir, '.repopickle_review' + )) + self._Unload() + + @property + def projects(self): + self._Load() + return self._projects + + def InitBranch(self): + m = self.manifestProject + if m.CurrentBranch is None: + b = m.revisionExpr + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + return m.StartBranch(b) + return True + + def SetMRefs(self, project): + if project.revisionId is None: + # Special project, e.g. the manifest or repo executable. + # + return + + ref = 'refs/remotes/m' + cur = project.bare_ref.get(ref) + exp = project.revisionId + if cur != exp: + msg = 'manifest set to %s' % exp + project.bare_git.UpdateRef(ref, exp, message = msg, detach = True) + + ref = 'refs/remotes/m-revision' + cur = project.bare_ref.symref(ref) + exp = project.revisionExpr + if exp is None: + if cur: + _rmref(project.gitdir, ref) + elif cur != exp: + remote = project.GetRemote(project.remote.name) + dst = remote.ToLocal(exp) + msg = 'manifest set to %s (%s)' % (exp, dst) + project.bare_git.symbolic_ref('-m', msg, ref, dst) + + def Upgrade_Local(self, old): + if isinstance(old, manifest_xml.XmlManifest): + self.FromXml_Local_1(old, checkout=True) + self.FromXml_Local_2(old) + else: + raise ManifestParseError, 'cannot upgrade manifest' + + def FromXml_Local_1(self, old, checkout): + os.rename(old.manifestProject.gitdir, + os.path.join(old.repodir, 'manifest.git')) + + oldmp = old.manifestProject + oldBranch = oldmp.CurrentBranch + b = oldmp.GetBranch(oldBranch).merge + if not b: + raise ManifestParseError, 'cannot upgrade manifest' + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + + newmp = self.manifestProject + self._CleanOldMRefs(newmp) + if oldBranch != b: + newmp.bare_git.branch('-m', oldBranch, b) + newmp.config.ClearCache() + + old_remote = newmp.GetBranch(b).remote.name + act_remote = self._GuessRemoteName(old) + if old_remote != act_remote: + newmp.bare_git.remote('rename', old_remote, act_remote) + newmp.config.ClearCache() + newmp.remote.name = act_remote + print >>sys.stderr, "Assuming remote named '%s'" % act_remote + + if checkout: + for p in old.projects.values(): + for c in p.copyfiles: + if os.path.exists(c.abs_dest): + os.remove(c.abs_dest) + newmp._InitWorkTree() + else: + newmp._LinkWorkTree() + + _lwrite(os.path.join(newmp.worktree,'.git',HEAD), + 'ref: refs/heads/%s\n' % b) + + def _GuessRemoteName(self, old): + used = {} + for p in old.projects.values(): + n = p.remote.name + used[n] = used.get(n, 0) + 1 + + remote_name = 'origin' + remote_used = 0 + for n in used.keys(): + if remote_used < used[n]: + remote_used = used[n] + remote_name = n + return remote_name + + def FromXml_Local_2(self, old): + shutil.rmtree(old.manifestProject.worktree) + os.remove(old._manifestFile) + + my_remote = self._Remote().name + new_base = os.path.join(self.repodir, 'projects') + old_base = os.path.join(self.repodir, 'projects.old') + os.rename(new_base, old_base) + os.makedirs(new_base) + + info = [] + pm = Progress('Converting projects', len(self.projects)) + for p in self.projects.values(): + pm.update() + + old_p = old.projects.get(p.name) + old_gitdir = os.path.join(old_base, '%s.git' % p.relpath) + if not os.path.isdir(old_gitdir): + continue + + parent = os.path.dirname(p.gitdir) + if not os.path.isdir(parent): + os.makedirs(parent) + os.rename(old_gitdir, p.gitdir) + _rmdir(os.path.dirname(old_gitdir), self.repodir) + + if not os.path.isdir(p.worktree): + os.makedirs(p.worktree) + + if os.path.isdir(os.path.join(p.worktree, '.git')): + p._LinkWorkTree(relink=True) + + self._CleanOldMRefs(p) + if old_p and old_p.remote.name != my_remote: + info.append("%s/: renamed remote '%s' to '%s'" \ + % (p.relpath, old_p.remote.name, my_remote)) + p.bare_git.remote('rename', old_p.remote.name, my_remote) + p.config.ClearCache() + + self.SetMRefs(p) + pm.end() + for i in info: + print >>sys.stderr, i + + def _CleanOldMRefs(self, p): + all_refs = p._allrefs + for ref in all_refs.keys(): + if ref.startswith(manifest_xml.R_M): + if p.bare_ref.symref(ref) != '': + _rmref(p.gitdir, ref) + else: + p.bare_git.DeleteRef(ref, all_refs[ref]) + + def FromXml_Definition(self, old): + """Convert another manifest representation to this one. + """ + mp = self.manifestProject + gm = self._modules + gr = self._review + + fd = open(os.path.join(mp.worktree, '.gitignore'), 'ab') + fd.write('/.repo\n') + fd.close() + + sort_projects = list(old.projects.keys()) + sort_projects.sort() + + b = mp.GetBranch(mp.CurrentBranch).merge + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + + info = [] + pm = Progress('Converting manifest', len(sort_projects)) + for p in sort_projects: + pm.update() + p = old.projects[p] + + gm.SetString('submodule.%s.path' % p.name, p.relpath) + gm.SetString('submodule.%s.url' % p.name, p.remote.url) + + if gr.GetString('review.url') is None: + gr.SetString('review.url', p.remote.review) + elif gr.GetString('review.url') != p.remote.review: + gr.SetString('review.%s.url' % p.name, p.remote.review) + + r = p.revisionExpr + if r and not IsId(r): + if r.startswith(R_HEADS): + r = r[len(R_HEADS):] + if r == b: + r = '.' + gm.SetString('submodule.%s.revision' % p.name, r) + + for c in p.copyfiles: + info.append('Moved %s out of %s' % (c.src, p.relpath)) + c._Copy() + p.work_git.rm(c.src) + mp.work_git.add(c.dest) + + self.SetRevisionId(p.relpath, p.GetRevisionId()) + mp.work_git.add('.gitignore', '.gitmodules', '.review') + pm.end() + for i in info: + print >>sys.stderr, i + + def _Unload(self): + self._loaded = False + self._projects = {} + self._revisionIds = None + self.branch = None + + def _Load(self): + if not self._loaded: + f = os.path.join(self.repodir, manifest_xml.LOCAL_MANIFEST_NAME) + if os.path.exists(f): + print >>sys.stderr, 'warning: ignoring %s' % f + + m = self.manifestProject + b = m.CurrentBranch + if not b: + raise ManifestParseError, 'manifest cannot be on detached HEAD' + b = m.GetBranch(b).merge + if b.startswith(R_HEADS): + b = b[len(R_HEADS):] + self.branch = b + m.remote.name = self._Remote().name + + self._ParseModules() + + if self.IsMirror: + self._AddMetaProjectMirror(self.repoProject) + self._AddMetaProjectMirror(self.manifestProject) + + self._loaded = True + + def _ParseModules(self): + byPath = dict() + for name in self._modules.GetSubSections('submodule'): + p = self._ParseProject(name) + if self._projects.get(p.name): + raise ManifestParseError, 'duplicate project "%s"' % p.name + if byPath.get(p.relpath): + raise ManifestParseError, 'duplicate path "%s"' % p.relpath + self._projects[p.name] = p + byPath[p.relpath] = p + + for relpath in self._allRevisionIds.keys(): + if relpath not in byPath: + raise ManifestParseError, \ + 'project "%s" not in .gitmodules' \ + % relpath + + def _Remote(self): + m = self.manifestProject + b = m.GetBranch(m.CurrentBranch) + return b.remote + + def _ResolveUrl(self, url): + if url.startswith('./') or url.startswith('../'): + base = self._Remote().url + try: + base = base[:base.rindex('/')+1] + except ValueError: + base = base[:base.rindex(':')+1] + if url.startswith('./'): + url = url[2:] + while '/' in base and url.startswith('../'): + base = base[:base.rindex('/')+1] + url = url[3:] + return base + url + return url + + def _GetRevisionId(self, path): + return self._allRevisionIds.get(path) + + @property + def _allRevisionIds(self): + if self._revisionIds is None: + a = dict() + p = GitCommand(self.manifestProject, + ['ls-files','-z','--stage'], + capture_stdout = True) + for line in p.process.stdout.read().split('\0')[:-1]: + l_info, l_path = line.split('\t', 2) + l_mode, l_id, l_stage = l_info.split(' ', 2) + if l_mode == GITLINK and l_stage == '0': + a[l_path] = l_id + p.Wait() + self._revisionIds = a + return self._revisionIds + + def SetRevisionId(self, path, id): + self.manifestProject.work_git.update_index( + '--add','--cacheinfo', GITLINK, id, path) + + def _ParseProject(self, name): + gm = self._modules + gr = self._review + + path = gm.GetString('submodule.%s.path' % name) + if not path: + path = name + + revId = self._GetRevisionId(path) + if not revId: + raise ManifestParseError( + 'submodule "%s" has no revision at "%s"' \ + % (name, path)) + + url = gm.GetString('submodule.%s.url' % name) + if not url: + url = name + url = self._ResolveUrl(url) + + review = gr.GetString('review.%s.url' % name) + if not review: + review = gr.GetString('review.url') + if not review: + review = self._Remote().review + + remote = RemoteSpec(self._Remote().name, url, review) + revExpr = gm.GetString('submodule.%s.revision' % name) + if revExpr == '.': + revExpr = self.branch + + if self.IsMirror: + relpath = None + worktree = None + gitdir = os.path.join(self.topdir, '%s.git' % name) + else: + worktree = os.path.join(self.topdir, path) + gitdir = os.path.join(self.repodir, 'projects/%s.git' % name) + + return Project(manifest = self, + name = name, + remote = remote, + gitdir = gitdir, + worktree = worktree, + relpath = path, + revisionExpr = revExpr, + revisionId = revId) + + def _AddMetaProjectMirror(self, m): + m_url = m.GetRemote(m.remote.name).url + if m_url.endswith('/.git'): + raise ManifestParseError, 'refusing to mirror %s' % m_url + + name = self._GuessMetaName(m_url) + if name.endswith('.git'): + name = name[:-4] + + if name not in self._projects: + m.PreSync() + gitdir = os.path.join(self.topdir, '%s.git' % name) + project = Project(manifest = self, + name = name, + remote = RemoteSpec(self._Remote().name, m_url), + gitdir = gitdir, + worktree = None, + relpath = None, + revisionExpr = m.revisionExpr, + revisionId = None) + self._projects[project.name] = project + + def _GuessMetaName(self, m_url): + parts = m_url.split('/') + name = parts[-1] + parts = parts[0:-1] + s = len(parts) - 1 + while s > 0: + l = '/'.join(parts[0:s]) + '/' + r = '/'.join(parts[s:]) + '/' + for p in self._projects.values(): + if p.name.startswith(r) and p.remote.url.startswith(l): + return r + name + s -= 1 + return m_url[m_url.rindex('/') + 1:] diff --git a/subcmds/init.py b/subcmds/init.py index b5207fbf..cdbbfdf7 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -21,6 +21,7 @@ from command import InteractiveCommand, MirrorSafeCommand from error import ManifestParseError from project import SyncBuffer from git_command import git_require, MIN_GIT_VERSION +from manifest_submodule import SubmoduleManifest from manifest_xml import XmlManifest from subcmds.sync import _ReloadManifest @@ -144,6 +145,14 @@ to update the working directory files. print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url sys.exit(1) + if is_new and SubmoduleManifest.IsBare(m): + new = self.GetManifest(reparse=True, type=SubmoduleManifest) + if m.gitdir != new.manifestProject.gitdir: + os.rename(m.gitdir, new.manifestProject.gitdir) + new = self.GetManifest(reparse=True, type=SubmoduleManifest) + m = new.manifestProject + self._ApplyOptions(opt, is_new) + if not is_new: # Force the manifest to load if it exists, the old graph # may be needed inside of _ReloadManifest(). diff --git a/subcmds/manifest.py b/subcmds/manifest.py index 551b13bd..7a8b2ee8 100644 --- a/subcmds/manifest.py +++ b/subcmds/manifest.py @@ -22,7 +22,7 @@ from manifest_xml import XmlManifest def _doc(name): r = os.path.dirname(__file__) r = os.path.dirname(r) - fd = open(os.path.join(r, 'docs', 'manifest_xml.txt')) + fd = open(os.path.join(r, 'docs', name)) try: return fd.read() finally: @@ -48,6 +48,8 @@ in a Git repository for use during future 'repo init' invocations. help = '' if isinstance(self.manifest, XmlManifest): help += self._xmlHelp + '\n' + _doc('manifest_xml.txt') + if isinstance(self.manifest, SubmoduleManifest): + help += _doc('manifest_submodule.txt') return help def _Options(self, p): From 57272ba82e3e1baa2bd7743d799f7dbc2acd43f8 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 3 Jul 2009 18:06:22 -0700 Subject: [PATCH 21/32] manifest: Support --upgrade to submodule format, from XML By running `repo manifest --uprade` an administrator can update the current manifest format from the XML format to submodule format, but we need all projects to be checked out in a work tree for this to function correctly. Signed-off-by: Shawn O. Pearce --- subcmds/manifest.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/subcmds/manifest.py b/subcmds/manifest.py index 7a8b2ee8..dcd3df17 100644 --- a/subcmds/manifest.py +++ b/subcmds/manifest.py @@ -17,6 +17,7 @@ import os import sys from command import PagedCommand +from manifest_submodule import SubmoduleManifest from manifest_xml import XmlManifest def _doc(name): @@ -54,6 +55,9 @@ in a Git repository for use during future 'repo init' invocations. def _Options(self, p): if isinstance(self.manifest, XmlManifest): + p.add_option('--upgrade', + dest='upgrade', action='store_true', + help='Upgrade XML manifest to submodule') p.add_option('-r', '--revision-as-HEAD', dest='peg_rev', action='store_true', help='Save revisions as current HEAD') @@ -62,6 +66,11 @@ in a Git repository for use during future 'repo init' invocations. help='File to save the manifest to', metavar='-|NAME.xml') + def WantPager(self, opt): + if isinstance(self.manifest, XmlManifest) and opt.upgrade: + return False + return True + def _Output(self, opt): if opt.output_file == '-': fd = sys.stdout @@ -73,12 +82,36 @@ in a Git repository for use during future 'repo init' invocations. if opt.output_file != '-': print >>sys.stderr, 'Saved manifest to %s' % opt.output_file + def _Upgrade(self): + old = self.manifest + + if isinstance(old, SubmoduleManifest): + print >>sys.stderr, 'error: already upgraded' + sys.exit(1) + + old._Load() + for p in old.projects.values(): + if not os.path.exists(p.gitdir) \ + or not os.path.exists(p.worktree): + print >>sys.stderr, 'fatal: project "%s" missing' % p.relpath + sys.exit(1) + + new = SubmoduleManifest(old.repodir) + new.FromXml_Local_1(old, checkout=False) + new.FromXml_Definition(old) + new.FromXml_Local_2(old) + print >>sys.stderr, 'upgraded manifest; commit result manually' + def Execute(self, opt, args): if args: self.Usage() - if isinstance(self.manifest, XmlManifest) \ - and opt.output_file is not None: + if isinstance(self.manifest, XmlManifest): + if opt.upgrade: + self._Upgrade() + return + + if opt.output_file is not None: self._Output(opt) return From 33f0e786bb35ea11da1e3d1746c431da5f1d93eb Mon Sep 17 00:00:00 2001 From: Mike Lockwood Date: Tue, 14 Jul 2009 15:23:39 -0400 Subject: [PATCH 22/32] Add "repo branch" as an alias for "repo branches" For those of us that are used to typing "git branch". Signed-off-by: Mike Lockwood --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index a60641d7..2cc7e447 100755 --- a/main.py +++ b/main.py @@ -59,6 +59,8 @@ class _Repo(object): def __init__(self, repodir): self.repodir = repodir self.commands = all_commands + # add 'branch' as an alias for 'branches' + all_commands['branch'] = all_commands['branches'] def _Run(self, argv): name = None From 4c0f67046543c7c6ab24175e143001f14b76ea01 Mon Sep 17 00:00:00 2001 From: Josh Guilfoyle Date: Sun, 16 Aug 2009 09:44:40 -0700 Subject: [PATCH 23/32] Do not invoke ssh with -p argument when no port has been specified. This change allows local SSH configuration to choose the port number to use when not explicitly set in the manifest. --- git_config.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/git_config.py b/git_config.py index e1e20463..fe983c4a 100644 --- a/git_config.py +++ b/git_config.py @@ -356,10 +356,14 @@ class RefSpec(object): _ssh_cache = {} _ssh_master = True -def _open_ssh(host, port): +def _open_ssh(host, port=None): global _ssh_master - key = '%s:%s' % (host, port) + if port is not None: + key = '%s:%s' % (host, port) + else: + key = host + if key in _ssh_cache: return True @@ -372,10 +376,13 @@ def _open_ssh(host, port): command = ['ssh', '-o','ControlPath %s' % _ssh_sock(), - '-p',str(port), '-M', '-N', host] + + if port is not None: + command[3:3] = ['-p',str(port)] + try: Trace(': %s', ' '.join(command)) p = subprocess.Popen(command) @@ -417,7 +424,7 @@ def _preconnect(url): if ':' in host: host, port = host.split(':') else: - port = 22 + port = None if scheme in ('ssh', 'git+ssh', 'ssh+git'): return _open_ssh(host, port) return False @@ -425,7 +432,7 @@ def _preconnect(url): m = URI_SCP.match(url) if m: host = m.group(1) - return _open_ssh(host, 22) + return _open_ssh(host) return False From a949fa5d202f0a1f812d7630f3e5bf0f02ca4e98 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 22 Aug 2009 18:17:46 -0700 Subject: [PATCH 24/32] Automatically install Gerrit Code Review's commit-msg hook Most users of repo are also using Gerrit Code Review, and will want the commit-msg hook to be automatically installed into their local projects so that Change-Ids are assigned when commits are created, not when they are first uploaded. Change-Id: Ide42e93b068832f099d68a79c2863d22145d05ad Signed-off-by: Shawn O. Pearce --- hooks/commit-msg | 44 ++++++++++++++++++++++++++++++++++++++++++++ project.py | 22 ++++++++++++++++++---- 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100755 hooks/commit-msg diff --git a/hooks/commit-msg b/hooks/commit-msg new file mode 100755 index 00000000..ecd6a20b --- /dev/null +++ b/hooks/commit-msg @@ -0,0 +1,44 @@ +#!/bin/sh + +MSG="$1" + +# Check for, and add if missing, a unique Change-Id +# +add_ChangeId() { + if grep '^Change-Id: ' "$MSG" >/dev/null + then + return + fi + + id=$(_gen_ChangeId) + out="$MSG.new" + ftt="$MSG.footers" + sed -e '/^[A-Za-z][A-Za-z0-9-]*: /,$d' <"$MSG" >"$out" + sed -ne '/^[A-Za-z][A-Za-z0-9-]*: /,$p' <"$MSG" >"$ftt" + if ! [ -s "$ftt" ] + then + echo >>"$out" + fi + echo "Change-Id: I$id" >>"$out" + cat "$ftt" >>"$out" + mv -f "$out" "$MSG" + rm -f "$out" "$ftt" +} +_gen_ChangeIdInput() { + echo "tree $(git write-tree)" + if parent=$(git rev-parse HEAD^0 2>/dev/null) + then + echo "parent $parent" + fi + echo "author $(git var GIT_AUTHOR_IDENT)" + echo "committer $(git var GIT_COMMITTER_IDENT)" + echo + cat "$MSG" +} +_gen_ChangeId() { + _gen_ChangeIdInput | + git hash-object -t commit --stdin +} + + +add_ChangeId diff --git a/project.py b/project.py index beacc92f..89f94f20 100644 --- a/project.py +++ b/project.py @@ -1056,13 +1056,27 @@ class Project(object): if not os.path.exists(hooks): os.makedirs(hooks) for stock_hook in repo_hooks(): - dst = os.path.join(hooks, os.path.basename(stock_hook)) + name = os.path.basename(stock_hook) + + if name in ('commit-msg') and not self.remote.review: + # Don't install a Gerrit Code Review hook if this + # project does not appear to use it for reviews. + # + continue + + dst = os.path.join(hooks, name) + if os.path.islink(dst): + continue + if os.path.exists(dst): + if filecmp.cmp(stock_hook, dst, shallow=False): + os.remove(dst) + else: + _error("%s: Not replacing %s hook", self.relpath, name) + continue try: os.symlink(relpath(stock_hook, dst), dst) except OSError, e: - if e.errno == errno.EEXIST: - pass - elif e.errno == errno.EPERM: + if e.errno == errno.EPERM: raise GitError('filesystem must support symlinks') else: raise From 47199010671a6724e18f061f4da63dcd46e3f354 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 22 Aug 2009 18:24:13 -0700 Subject: [PATCH 25/32] upload: Document --replace is deprecated Change-Id: I52715bcfec9c038d0e02505aa7e4054ebc0434aa Signed-off-by: Shawn O. Pearce --- subcmds/upload.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/subcmds/upload.py b/subcmds/upload.py index aea399b6..8e3d2723 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -55,11 +55,11 @@ added to the respective list of users, and emails are sent to any new users. Users passed as --reviewers must already be registered with the code review system, or the upload will fail. -If the --replace option is passed the user can designate which -existing change(s) in Gerrit match up to the commits in the branch -being uploaded. For each matched pair of change,commit the commit -will be added as a new patch set, completely replacing the set of -files and description associated with the change in Gerrit. +If the --replace option (deprecated) is passed the user can designate +which existing change(s) in Gerrit match up to the commits in the +branch being uploaded. For each matched pair of change,commit the +commit will be added as a new patch set, completely replacing the +set of files and description associated with the change in Gerrit. Configuration ------------- @@ -92,7 +92,7 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ def _Options(self, p): p.add_option('--replace', dest='replace', action='store_true', - help='Upload replacement patchesets from this branch') + help='Upload replacement patchsets from this branch (deprecated)') p.add_option('--re', '--reviewers', type='string', action='append', dest='reviewers', help='Request reviews from these people.') From d2dfac81ad6a060179b4b2289060af2dc7a5cdfd Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 22 Aug 2009 18:39:49 -0700 Subject: [PATCH 26/32] upload: Catch and cleanly report connectivity errors Instead of giving a Python backtrace when there is a connectivity problem during repo upload, report that we cannot access the host, and why, with a halfway decent error message. Bug: REPO-45 Change-Id: I9a45b387e86e48073a2d99bd6d594c1a7d6d99d4 Signed-off-by: Shawn O. Pearce --- git_config.py | 10 +++++++--- subcmds/upload.py | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/git_config.py b/git_config.py index 9dba699a..b6288219 100644 --- a/git_config.py +++ b/git_config.py @@ -19,6 +19,8 @@ import re import subprocess import sys import time +import urllib2 + from signal import SIGTERM from urllib2 import urlopen, HTTPError from error import GitError, UploadError @@ -487,23 +489,25 @@ class Remote(object): try: info = urlopen(u).read() if info == 'NOT_AVAILABLE': - raise UploadError('Upload over ssh unavailable') + raise UploadError('%s: SSH disabled' % self.review) if '<' in info: # Assume the server gave us some sort of HTML # response back, like maybe a login page. # - raise UploadError('Cannot read %s:\n%s' % (u, info)) + raise UploadError('%s: Cannot parse response' % u) self._review_protocol = 'ssh' self._review_host = info.split(" ")[0] self._review_port = info.split(" ")[1] + except urllib2.URLError, e: + raise UploadError('%s: %s' % (self.review, e.reason[1])) except HTTPError, e: if e.code == 404: self._review_protocol = 'http-post' self._review_host = None self._review_port = None else: - raise UploadError('Cannot guess Gerrit version') + raise UploadError('Upload over ssh unavailable') REVIEW_CACHE[u] = ( self._review_protocol, diff --git a/subcmds/upload.py b/subcmds/upload.py index 8e3d2723..2ab6a484 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -273,15 +273,19 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ have_errors = True print >>sys.stderr, '' - print >>sys.stderr, '--------------------------------------------' + print >>sys.stderr, '----------------------------------------------------------------------' if have_errors: for branch in todo: if not branch.uploaded: - print >>sys.stderr, '[FAILED] %-15s %-15s (%s)' % ( + if len(str(branch.error)) <= 30: + fmt = ' (%s)' + else: + fmt = '\n (%s)' + print >>sys.stderr, ('[FAILED] %-15s %-15s' + fmt) % ( branch.project.relpath + '/', \ branch.name, \ - branch.error) + str(branch.error)) print >>sys.stderr, '' for branch in todo: From d4cd69bdef28c5a9287c85c48a18ce621eba689d Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 22 Aug 2009 18:50:45 -0700 Subject: [PATCH 27/32] forall: Silently skip missing projects If a project is missing locally, it might be OK to skip over it and continue running the same command in other projects. Bug: REPO-43 Change-Id: I64f97eb315f379ab2c51fc53d24ed340b3d09250 Signed-off-by: Shawn O. Pearce --- subcmds/forall.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/subcmds/forall.py b/subcmds/forall.py index b66313d7..6bd867e7 100644 --- a/subcmds/forall.py +++ b/subcmds/forall.py @@ -169,6 +169,12 @@ terminal and are not redirected. else: cwd = project.worktree + if not os.path.exists(cwd): + if (opt.project_header and opt.verbose) \ + or not opt.project_header: + print >>sys.stderr, 'skipping %s/' % project.relpath + continue + if opt.project_header: stdin = subprocess.PIPE stdout = subprocess.PIPE From 15f6579eb3a2f3cb0e432ed39ac39d8678786161 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 22 Aug 2009 19:23:55 -0700 Subject: [PATCH 28/32] commit-msg: Update the commit message hook This version fixes a bug where Change-Id lines become the subject line, if the subject used a pattern like the subject of this message does. Change-Id: I7f7e0363091d03eb05dead2992fc19763214de65 Signed-off-by: Shawn O. Pearce --- hooks/commit-msg | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/hooks/commit-msg b/hooks/commit-msg index ecd6a20b..5b1fe3ae 100755 --- a/hooks/commit-msg +++ b/hooks/commit-msg @@ -1,4 +1,22 @@ #!/bin/sh +# From Gerrit Code Review v2.0.18-100-g98fc90a +# +# Part of Gerrit Code Review (http://code.google.com/p/gerrit/) +# +# Copyright (C) 2009 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# MSG="$1" @@ -13,8 +31,12 @@ add_ChangeId() { id=$(_gen_ChangeId) out="$MSG.new" ftt="$MSG.footers" - sed -e '/^[A-Za-z][A-Za-z0-9-]*: /,$d' <"$MSG" >"$out" - sed -ne '/^[A-Za-z][A-Za-z0-9-]*: /,$p' <"$MSG" >"$ftt" + sed -e '2,${ + /^[A-Za-z][A-Za-z0-9-]*: /,$d + }' <"$MSG" >"$out" + sed -ne '2,${ + /^[A-Za-z][A-Za-z0-9-]*: /,$p + }' <"$MSG" >"$ftt" if ! [ -s "$ftt" ] then echo >>"$out" From c024912fb8227db2a1c9232add3b1e7256af170a Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 25 Aug 2009 11:38:11 -0700 Subject: [PATCH 29/32] commit-msg: Don't create message with only Change-Id If a user aborts a commit, the commit-msg hook is still called, but with an empty file. We need to leave the empty file alone. Change-Id: I13766135dac267823cb08ab76f67d2000ba2d1ce Signed-off-by: Shawn O. Pearce --- hooks/commit-msg | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/hooks/commit-msg b/hooks/commit-msg index 5b1fe3ae..fd76c074 100755 --- a/hooks/commit-msg +++ b/hooks/commit-msg @@ -1,5 +1,5 @@ #!/bin/sh -# From Gerrit Code Review v2.0.18-100-g98fc90a +# From Gerrit Code Review v2.0.19.1-4-g21d307b # # Part of Gerrit Code Review (http://code.google.com/p/gerrit/) # @@ -23,21 +23,34 @@ MSG="$1" # Check for, and add if missing, a unique Change-Id # add_ChangeId() { - if grep '^Change-Id: ' "$MSG" >/dev/null + clean_message=$(sed -e ' + /^diff --git a\/.*/{ + s/// + q + } + /^Signed-off-by:/d + /^#/d + ' "$MSG" | git stripspace) + if test -z "$clean_message" + then + return + fi + + if grep -i '^Change-Id:' "$MSG" >/dev/null then return fi id=$(_gen_ChangeId) - out="$MSG.new" - ftt="$MSG.footers" + out="$MSG.OUT" + ftt="$MSG.FTT" sed -e '2,${ /^[A-Za-z][A-Za-z0-9-]*: /,$d }' <"$MSG" >"$out" sed -ne '2,${ /^[A-Za-z][A-Za-z0-9-]*: /,$p }' <"$MSG" >"$ftt" - if ! [ -s "$ftt" ] + if ! test -s "$ftt" then echo >>"$out" fi @@ -55,7 +68,7 @@ _gen_ChangeIdInput() { echo "author $(git var GIT_AUTHOR_IDENT)" echo "committer $(git var GIT_COMMITTER_IDENT)" echo - cat "$MSG" + printf '%s' "$clean_message" } _gen_ChangeId() { _gen_ChangeIdInput | From 840ed0fab7cb4c2ab296c7d7d45f13e2523bae1c Mon Sep 17 00:00:00 2001 From: Thiago Farina Date: Wed, 9 Sep 2009 00:41:34 -0400 Subject: [PATCH 30/32] Fix to display the usage message of the command download when the user don't provide any arguments to 'repo download'. Signed-off-by: Thiago Farina --- subcmds/download.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/subcmds/download.py b/subcmds/download.py index a6f3aa45..61eadd54 100644 --- a/subcmds/download.py +++ b/subcmds/download.py @@ -36,6 +36,9 @@ makes it available in your project's local working directory. pass def _ParseChangeIds(self, args): + if not args: + self.Usage() + to_get = [] project = None From b0f9a02394779c1c9422a9649412c9ac5fb0f12f Mon Sep 17 00:00:00 2001 From: Anthony Newnam Date: Mon, 29 Nov 2010 13:15:24 -0600 Subject: [PATCH 31/32] Make path references OS independent Change-Id: I5573995adfd52fd54bddc62d1d1ea78fb1328130 --- command.py | 6 ++++-- manifest_xml.py | 2 +- project.py | 4 ++-- repo | 6 +++++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/command.py b/command.py index 6e4e2c57..4e0253fc 100644 --- a/command.py +++ b/command.py @@ -90,7 +90,7 @@ class Command(object): project = all.get(arg) if not project: - path = os.path.abspath(arg) + path = os.path.abspath(arg).replace('\\', '/') if not by_path: by_path = dict() @@ -100,13 +100,15 @@ class Command(object): try: project = by_path[path] except KeyError: + oldpath = None while path \ - and path != '/' \ + and path != oldpath \ and path != self.manifest.topdir: try: project = by_path[path] break except KeyError: + oldpath = path path = os.path.dirname(path) if not project: diff --git a/manifest_xml.py b/manifest_xml.py index d888653d..35318d0a 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -357,7 +357,7 @@ class XmlManifest(Manifest): worktree = None gitdir = os.path.join(self.topdir, '%s.git' % name) else: - worktree = os.path.join(self.topdir, path) + worktree = os.path.join(self.topdir, path).replace('\\', '/') gitdir = os.path.join(self.repodir, 'projects/%s.git' % path) project = Project(manifest = self, diff --git a/project.py b/project.py index 5a143a76..1cea959e 100644 --- a/project.py +++ b/project.py @@ -233,8 +233,8 @@ class Project(object): self.manifest = manifest self.name = name self.remote = remote - self.gitdir = gitdir - self.worktree = worktree + self.gitdir = gitdir.replace('\\', '/') + self.worktree = worktree.replace('\\', '/') self.relpath = relpath self.revisionExpr = revisionExpr diff --git a/repo b/repo index ff7c4188..3a545cc6 100755 --- a/repo +++ b/repo @@ -432,10 +432,14 @@ def _FindRepo(): dir = os.getcwd() repo = None - while dir != '/' and not repo: + olddir = None + while dir != '/' \ + and dir != olddir \ + and not repo: repo = os.path.join(dir, repodir, REPO_MAIN) if not os.path.isfile(repo): repo = None + olddir = dir dir = os.path.dirname(dir) return (repo, os.path.join(dir, repodir)) From 3218c13205694434edb2375ab8a8515554eed366 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 7 Dec 2010 08:46:14 -0800 Subject: [PATCH 32/32] Use os.environ.copy() instead of dict() Signed-off-by: Shawn O. Pearce --- git_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git_command.py b/git_command.py index d56ad0a8..414c84a2 100644 --- a/git_command.py +++ b/git_command.py @@ -104,7 +104,7 @@ class GitCommand(object): ssh_proxy = False, cwd = None, gitdir = None): - env = dict(os.environ) + env = os.environ.copy() for e in [REPO_TRACE, GIT_DIR,