Document any crashes from the user's text editor

Rather than failing with no information, display the child exit
status and the command line we tried to use to edit a text file.
There may be some useful information to help understand the crash.

Signed-off-by: Shawn O. Pearce <sop@google.com>

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>repo</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/repo</path>
+</pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.4</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+</pydev_project>

docs/manifest-format.txt

@@ -23,32 +23,23 @@ following DTD:
     <!ELEMENT manifest (remote*,
                         default?,
                         remove-project*,
-                        project*,
-                        add-remote*)>
+                        project*)>
   
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
-    <!ATTLIST remote project-name CDATA #IMPLIED>
   
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote   IDREF #IMPLIED>
     <!ATTLIST default revision CDATA #IMPLIED>
   
-    <!ELEMENT project (remote*)>
+    <!ELEMENT project (EMPTY)>
     <!ATTLIST project name     CDATA #REQUIRED>
     <!ATTLIST project path     CDATA #IMPLIED>
     <!ATTLIST project remote   IDREF #IMPLIED>
     <!ATTLIST project revision CDATA #IMPLIED>
   
-    <!ELEMENT add-remote (EMPTY)>
-    <!ATTLIST add-remote to-project   ID    #REQUIRED>
-    <!ATTLIST add-remote name         ID    #REQUIRED>
-    <!ATTLIST add-remote fetch        CDATA #REQUIRED>
-    <!ATTLIST add-remote review       CDATA #IMPLIED>
-    <!ATTLIST add-remote project-name CDATA #IMPLIED>
-  
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
   ]>
@@ -82,25 +73,6 @@ Attribute `review`: Hostname of the Gerrit server where reviews
 are uploaded to by `repo upload`.  This attribute is optional;
 if not specified then `repo upload` will not function.
 
-Attribute `project-name`: Specifies the name of this project used
-by the review server given in the review attribute of this element.
-Only permitted when the remote element is nested inside of a project
-element (see below).  If not given, defaults to the name supplied
-in the project's name attribute.
-
-Element add-remote
-------------------
-
-Adds a remote to an existing project, whose name is given by the
-to-project attribute.  This is functionally equivalent to nesting
-a remote element under the project, but has the advantage that it
-can be specified in the uesr's `local_manifest.xml` to add a remote
-to a project declared by the normal manifest.
-
-The element can be used to add a fork of an existing project that
-the user needs to work with.
-
-
 Element default
 ---------------
 
@@ -152,13 +124,6 @@ Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
 the default element is used.
 
-Child element `remote`: Described like the top-level remote element,
-but adds an additional remote to only this project.  These additional
-remotes are fetched from first on the initial `repo sync`, causing
-the majority of the project's object database to be obtained through
-these additional remotes.
-
-
 Element remove-project
 ----------------------
 

editor.py

@@ -76,8 +76,15 @@ least one of these before using this command."""
       os.close(fd)
       fd = None
 
-      if subprocess.Popen(editor + [path]).wait() != 0:
-        raise EditorError()
+      try:
+        rc = subprocess.Popen(editor + [path]).wait()
+      except OSError, e:
+        raise EditorError('editor failed, %s: %s %s'
+          % (str(e), cls._GetEditor(), path))
+      if rc != 0:
+        raise EditorError('editor failed with exit status %d: %s %s'
+          % (rc, cls._GetEditor(), path))
+
       fd2 = open(path)
       try:
         return fd2.read()

error.py

@@ -24,6 +24,11 @@ class ManifestInvalidRevisionError(Exception):
 class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """
+  def __init__(self, reason):
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
 
 class GitError(Exception):
   """Unspecified internal error from git.

git_command.py

@@ -68,6 +68,30 @@ class _GitCall(object):
     return fun
 git = _GitCall()
 
+_git_version = None
+
+def git_require(min_version, fail=False):
+  global _git_version
+
+  if _git_version is None:
+    ver_str = git.version()
+    if ver_str.startswith('git version '):
+      _git_version = tuple(
+        map(lambda x: int(x),
+          ver_str[len('git version '):].strip().split('.')[0:3]
+        ))
+    else:
+      print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
+      sys.exit(1)
+
+  if min_version <= _git_version:
+    return True
+  if fail:
+    need = '.'.join(map(lambda x: str(x), min_version))
+    print >>sys.stderr, 'fatal: git %s or later required' % need
+    sys.exit(1)
+  return False
+
 class GitCommand(object):
   def __init__(self,
                project,

git_config.py

@@ -56,16 +56,20 @@ class GitConfig(object):
     return cls(file = os.path.join(gitdir, 'config'),
                defaults = defaults)
 
-  def __init__(self, file, defaults=None):
+  def __init__(self, file, defaults=None, pickleFile=None):
     self.file = file
     self.defaults = defaults
     self._cache_dict = None
     self._section_dict = None
     self._remotes = {}
     self._branches = {}
-    self._pickle = os.path.join(
-      os.path.dirname(self.file),
-      '.repopickle_' + os.path.basename(self.file))
+
+    if pickleFile is None:
+      self._pickle = os.path.join(
+        os.path.dirname(self.file),
+        '.repopickle_' + os.path.basename(self.file))
+    else:
+      self._pickle = pickleFile
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
@@ -172,6 +176,11 @@ class GitConfig(object):
       self._branches[b.name] = b
     return b
 
+  def GetSubSections(self, section):
+    """List all subsection names matching $section.*.*
+    """
+    return self._sections.get(section, set())
+
   def HasSection(self, section, subsection = ''):
     """Does at least one key in section.subsection exist?
     """
@@ -227,6 +236,9 @@ class GitConfig(object):
         return cPickle.load(fd)
       finally:
         fd.close()
+    except EOFError:
+      os.remove(self._pickle)
+      return None
     except IOError:
       os.remove(self._pickle)
       return None
@@ -373,8 +385,11 @@ def _open_ssh(host, port):
 
 def close_ssh():
   for key,p in _ssh_cache.iteritems():
-    os.kill(p.pid, SIGTERM)
-    p.wait()
+    try:
+      os.kill(p.pid, SIGTERM)
+      p.wait()
+    except OSError:
+      pass
   _ssh_cache.clear()
 
   d = _ssh_sock(create=False)
@@ -405,6 +420,7 @@ def _preconnect(url):
     host = m.group(1)
     return _open_ssh(host, 22)
 
+  return False
 
 class Remote(object):
   """Configuration options related to a remote.

main.py

@@ -36,7 +36,7 @@ from editor import Editor
 from error import ManifestInvalidRevisionError
 from error import NoSuchProjectError
 from error import RepoChangedException
-from manifest import Manifest
+from manifest_xml import XmlManifest
 from pager import RunPager
 
 from subcmds import all as all_commands
@@ -97,7 +97,7 @@ class _Repo(object):
       sys.exit(1)
 
     cmd.repodir = self.repodir
-    cmd.manifest = Manifest(cmd.repodir)
+    cmd.manifest = XmlManifest(cmd.repodir)
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:

manifest_xml.py

@@ -18,8 +18,7 @@ import sys
 import xml.dom.minidom
 
 from git_config import GitConfig, IsId
-from project import Project, MetaProject, R_HEADS, HEAD
-from remote import Remote
+from project import RemoteSpec, Project, MetaProject, R_HEADS, HEAD
 from error import ManifestParseError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
@@ -28,11 +27,26 @@ LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 class _Default(object):
   """Project defaults within the manifest."""
 
-  revision = None
+  revisionExpr = None
   remote = None
 
+class _XmlRemote(object):
+  def __init__(self,
+               name,
+               fetch=None,
+               review=None):
+    self.name = name
+    self.fetchUrl = fetch
+    self.reviewUrl = review
 
-class Manifest(object):
+  def ToRemoteSpec(self, projectName):
+    url = self.fetchUrl
+    while url.endswith('/'):
+      url = url[:-1]
+    url += '/%s.git' % projectName
+    return RemoteSpec(self.name, url, self.reviewUrl)
+
+class XmlManifest(object):
   """manages the repo configuration file"""
 
   def __init__(self, repodir):
@@ -80,8 +94,6 @@ class Manifest(object):
     e.setAttribute('fetch', r.fetchUrl)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
-    if r.projectName is not None:
-      e.setAttribute('project-name', r.projectName)
 
   def Save(self, fd, peg_rev=False):
     """Write the current manifest out to the given file descriptor.
@@ -104,9 +116,9 @@ class Manifest(object):
     if d.remote:
       have_default = True
       e.setAttribute('remote', d.remote.name)
-    if d.revision:
+    if d.revisionExpr:
       have_default = True
-      e.setAttribute('revision', d.revision)
+      e.setAttribute('revision', d.revisionExpr)
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -126,15 +138,13 @@ class Manifest(object):
       if peg_rev:
         if self.IsMirror:
           e.setAttribute('revision',
-                         p.bare_git.rev_parse(p.revision + '^0'))
+                         p.bare_git.rev_parse(p.revisionExpr + '^0'))
         else:
           e.setAttribute('revision',
                          p.work_git.rev_parse(HEAD + '^0'))
-      elif not d.revision or p.revision != d.revision:
-        e.setAttribute('revision', p.revision)
+      elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
+        e.setAttribute('revision', p.revisionExpr)
 
-      for r in p.extraRemotes:
-        self._RemoteToXml(p.extraRemotes[r], doc, e)
       for c in p.copyfiles:
         ce = doc.createElement('copyfile')
         ce.setAttribute('src', c.src)
@@ -245,16 +255,6 @@ class Manifest(object):
                 (project.name, self.manifestFile)
         self._projects[project.name] = project
 
-    for node in config.childNodes:
-      if node.nodeName == 'add-remote':
-        pn = self._reqatt(node, 'to-project')
-        project = self._projects.get(pn)
-        if not project:
-          raise ManifestParseError, \
-                'project %s not defined in %s' % \
-                (pn, self.manifestFile)
-        self._ParseProjectExtraRemote(project, node)
-
   def _AddMetaProjectMirror(self, m):
     name = None
     m_url = m.GetRemote(m.remote.name).url
@@ -271,7 +271,7 @@ class Manifest(object):
 
     if name is None:
       s = m_url.rindex('/') + 1
-      remote = Remote('origin', fetch = m_url[:s])
+      remote = _XmlRemote('origin', m_url[:s])
       name = m_url[s:]
 
     if name.endswith('.git'):
@@ -282,11 +282,12 @@ class Manifest(object):
       gitdir = os.path.join(self.topdir, '%s.git' % name)
       project = Project(manifest = self,
                         name = name,
-                        remote = remote,
+                        remote = remote.ToRemoteSpec(name),
                         gitdir = gitdir,
                         worktree = None,
                         relpath = None,
-                        revision = m.revision)
+                        revisionExpr = m.revisionExpr,
+                        revisionId = None)
       self._projects[project.name] = project
 
   def _ParseRemote(self, node):
@@ -298,21 +299,7 @@ class Manifest(object):
     review = node.getAttribute('review')
     if review == '':
       review = None
-
-    projectName = node.getAttribute('project-name')
-    if projectName == '':
-      projectName = None
-
-    r = Remote(name=name,
-               fetch=fetch,
-               review=review,
-               projectName=projectName)
-
-    for n in node.childNodes:
-      if n.nodeName == 'require':
-        r.requiredCommits.append(self._reqatt(n, 'commit'))
-
-    return r
+    return _XmlRemote(name, fetch, review)
 
   def _ParseDefault(self, node):
     """
@@ -320,9 +307,9 @@ class Manifest(object):
     """
     d = _Default()
     d.remote = self._get_remote(node)
-    d.revision = node.getAttribute('revision')
-    if d.revision == '':
-      d.revision = None
+    d.revisionExpr = node.getAttribute('revision')
+    if d.revisionExpr == '':
+      d.revisionExpr = None
     return d
 
   def _ParseProject(self, node):
@@ -339,10 +326,10 @@ class Manifest(object):
             "no remote for project %s within %s" % \
             (name, self.manifestFile)
 
-    revision = node.getAttribute('revision')
-    if not revision:
-      revision = self._default.revision
-    if not revision:
+    revisionExpr = node.getAttribute('revision')
+    if not revisionExpr:
+      revisionExpr = self._default.revisionExpr
+    if not revisionExpr:
       raise ManifestParseError, \
             "no revision for project %s within %s" % \
             (name, self.manifestFile)
@@ -365,29 +352,19 @@ class Manifest(object):
 
     project = Project(manifest = self,
                       name = name,
-                      remote = remote,
+                      remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
                       worktree = worktree,
                       relpath = path,
-                      revision = revision)
+                      revisionExpr = revisionExpr,
+                      revisionId = None)
 
     for n in node.childNodes:
-      if n.nodeName == 'remote':
-        self._ParseProjectExtraRemote(project, n)
-      elif n.nodeName == 'copyfile':
+      if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
 
     return project
 
-  def _ParseProjectExtraRemote(self, project, n):
-    r = self._ParseRemote(n)
-    if project.extraRemotes.get(r.name) \
-       or project.remote.name == r.name:
-      raise ManifestParseError, \
-            'duplicate remote %s in project %s in %s' % \
-            (r.name, project.name, self.manifestFile)
-    project.extraRemotes[r.name] = r
-
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
     dest = self._reqatt(node, 'dest')

project.py

@@ -26,7 +26,6 @@ from git_command import GitCommand
 from git_config import GitConfig, IsId
 from error import GitError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
-from remote import Remote
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
@@ -212,6 +211,14 @@ class _CopyFile:
       except IOError:
         _error('Cannot copy file %s to %s', src, dest)
 
+class RemoteSpec(object):
+  def __init__(self,
+               name,
+               url = None,
+               review = None):
+    self.name = name
+    self.url = url
+    self.review = review
 
 class Project(object):
   def __init__(self,
@@ -221,16 +228,24 @@ class Project(object):
                gitdir,
                worktree,
                relpath,
-               revision):
+               revisionExpr,
+               revisionId):
     self.manifest = manifest
     self.name = name
     self.remote = remote
     self.gitdir = gitdir
     self.worktree = worktree
     self.relpath = relpath
-    self.revision = revision
+    self.revisionExpr = revisionExpr
+
+    if   revisionId is None \
+     and revisionExpr \
+     and IsId(revisionExpr):
+      self.revisionId = revisionExpr
+    else:
+      self.revisionId = revisionId
+
     self.snapshots = {}
-    self.extraRemotes = {}
     self.copyfiles = []
     self.config = GitConfig.ForRepository(
                     gitdir = self.gitdir,
@@ -579,9 +594,6 @@ class Project(object):
       self._InitGitDir()
 
     self._InitRemote()
-    for r in self.extraRemotes.values():
-      if not self._RemoteFetch(r.name):
-        return False
     if not self._RemoteFetch():
       return False
 
@@ -602,6 +614,23 @@ class Project(object):
     for file in self.copyfiles:
       file._Copy()
 
+  def GetRevisionId(self, all=None):
+    if self.revisionId:
+      return self.revisionId
+
+    rem = self.GetRemote(self.remote.name)
+    rev = rem.ToLocal(self.revisionExpr)
+
+    if all is not None and rev in all:
+      return all[rev]
+
+    try:
+      return self.bare_git.rev_parse('--verify', '%s^0' % rev)
+    except GitError:
+      raise ManifestInvalidRevisionError(
+        'revision %s in %s not found' % (self.revisionExpr,
+                                         self.name))
+
   def Sync_LocalHalf(self, syncbuf):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
@@ -610,19 +639,7 @@ class Project(object):
     all = self.bare_ref.all
     self.CleanPublishedCache(all)
 
-    rem = self.GetRemote(self.remote.name)
-    rev = rem.ToLocal(self.revision)
-    if rev in all:
-      revid = all[rev]
-    elif IsId(rev):
-      revid = rev
-    else:
-      try:
-        revid = self.bare_git.rev_parse('--verify', '%s^0' % rev)
-      except GitError:
-        raise ManifestInvalidRevisionError(
-          'revision %s in %s not found' % (self.revision, self.name))
-
+    revid = self.GetRevisionId(all)
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       branch = head[len(R_HEADS):]
@@ -646,11 +663,11 @@ class Project(object):
         #
         return
 
-      lost = self._revlist(not_rev(rev), HEAD)
+      lost = self._revlist(not_rev(revid), HEAD)
       if lost:
         syncbuf.info(self, "discarding %d commits", len(lost))
       try:
-        self._Checkout(rev, quiet=True)
+        self._Checkout(revid, quiet=True)
       except GitError, e:
         syncbuf.fail(self, e)
         return
@@ -663,9 +680,8 @@ class Project(object):
       return
 
     branch = self.GetBranch(branch)
-    merge = branch.LocalMerge
 
-    if not merge:
+    if not branch.LocalMerge:
       # The current branch has no tracking configuration.
       # Jump off it to a deatched HEAD.
       #
@@ -673,17 +689,17 @@ class Project(object):
                    "leaving %s; does not track upstream",
                    branch.name)
       try:
-        self._Checkout(rev, quiet=True)
+        self._Checkout(revid, quiet=True)
       except GitError, e:
         syncbuf.fail(self, e)
         return
       self._CopyFiles()
       return
 
-    upstream_gain = self._revlist(not_rev(HEAD), rev)
+    upstream_gain = self._revlist(not_rev(HEAD), revid)
     pub = self.WasPublished(branch.name, all)
     if pub:
-      not_merged = self._revlist(not_rev(rev), pub)
+      not_merged = self._revlist(not_rev(revid), pub)
       if not_merged:
         if upstream_gain:
           # The user has published this branch and some of those
@@ -700,69 +716,70 @@ class Project(object):
         # strict subset.  We can fast-forward safely.
         #
         def _doff():
-          self._FastForward(rev)
+          self._FastForward(revid)
           self._CopyFiles()
         syncbuf.later1(self, _doff)
         return
 
-    if merge == rev:
-      try:
-        old_merge = self.bare_git.rev_parse('%s@{1}' % merge)
-      except GitError:
-        old_merge = merge
-      if old_merge == '0000000000000000000000000000000000000000' \
-         or old_merge == '':
-        old_merge = merge
-    else:
-      # The upstream switched on us.  Time to cross our fingers
-      # and pray that the old upstream also wasn't in the habit
-      # of rebasing itself.
-      #
-      syncbuf.info(self, "manifest switched %s...%s", merge, rev)
-      old_merge = merge
+    # Examine the local commits not in the remote.  Find the
+    # last one attributed to this user, if any.
+    #
+    local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
+    last_mine = None
+    cnt_mine = 0
+    for commit in local_changes:
+      commit_id, committer_email = commit.split(' ', 2)
+      if committer_email == self.UserEmail:
+        last_mine = commit_id
+        cnt_mine += 1
 
-    if rev == old_merge:
-      upstream_lost = []
-    else:
-      upstream_lost = self._revlist(not_rev(rev), old_merge)
-
-    if not upstream_lost and not upstream_gain:
-      # Trivially no changes caused by the upstream.
-      #
+    if not upstream_gain and cnt_mine == len(local_changes):
       return
 
     if self.IsDirty(consider_untracked=False):
       syncbuf.fail(self, _DirtyError())
       return
 
-    if upstream_lost:
+    # If the upstream switched on us, warn the user.
+    #
+    if branch.merge != self.revisionExpr:
+      if branch.merge and self.revisionExpr:
+        syncbuf.info(self,
+                     'manifest switched %s...%s',
+                     branch.merge,
+                     self.revisionExpr)
+      elif branch.merge:
+        syncbuf.info(self,
+                     'manifest no longer tracks %s',
+                     branch.merge)
+
+    if cnt_mine < len(local_changes):
       # Upstream rebased.  Not everything in HEAD
-      # may have been caused by the user.
+      # was created by this user.
       #
       syncbuf.info(self,
                    "discarding %d commits removed from upstream",
-                   len(upstream_lost))
+                   len(local_changes) - cnt_mine)
 
-    branch.remote = rem
-    branch.merge = self.revision
+    branch.remote = self.GetRemote(self.remote.name)
+    branch.merge = self.revisionExpr
     branch.Save()
 
-    my_changes = self._revlist(not_rev(old_merge), HEAD)
-    if my_changes:
+    if cnt_mine > 0:
       def _dorebase():
-        self._Rebase(upstream = old_merge, onto = rev)
+        self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
         self._CopyFiles()
       syncbuf.later2(self, _dorebase)
-    elif upstream_lost:
+    elif local_changes:
       try:
-        self._ResetHard(rev)
+        self._ResetHard(revid)
         self._CopyFiles()
       except GitError, e:
         syncbuf.fail(self, e)
         return
     else:
       def _doff():
-        self._FastForward(rev)
+        self._FastForward(revid)
         self._CopyFiles()
       syncbuf.later1(self, _doff)
 
@@ -784,7 +801,7 @@ class Project(object):
     if GitCommand(self, cmd, bare=True).Wait() != 0:
       return None
     return DownloadedChange(self,
-                            remote.ToLocal(self.revision),
+                            self.GetRevisionId(),
                             change_id,
                             patch_id,
                             self.bare_git.rev_parse('FETCH_HEAD'))
@@ -808,15 +825,8 @@ class Project(object):
 
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revision
-
-    rev = branch.LocalMerge
-    if rev in all:
-      revid = all[rev]
-    elif IsId(rev):
-      revid = rev
-    else:
-      revid = None
+    branch.merge = self.revisionExpr
+    revid = self.GetRevisionId(all)
 
     if head.startswith(R_HEADS):
       try:
@@ -837,7 +847,7 @@ class Project(object):
       return True
 
     if GitCommand(self,
-                  ['checkout', '-b', branch.name, rev],
+                  ['checkout', '-b', branch.name, revid],
                   capture_stdout = True,
                   capture_stderr = True).Wait() == 0:
       branch.Save()
@@ -898,19 +908,12 @@ class Project(object):
       #
       head = all[head]
 
-      rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-      if rev in all:
-        revid = all[rev]
-      elif IsId(rev):
-        revid = rev
-      else:
-        revid = None
-
-      if revid and head == revid:
+      revid = self.GetRevisionId(all)
+      if head == revid:
         _lwrite(os.path.join(self.worktree, '.git', HEAD),
                 '%s\n' % revid)
       else:
-        self._Checkout(rev, quiet=True)
+        self._Checkout(revid, quiet=True)
 
     return GitCommand(self,
                       ['branch', '-D', name],
@@ -929,7 +932,7 @@ class Project(object):
         if cb is None or name != cb:
           kill.append(name)
 
-    rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
+    rev = self.GetRevisionId(left)
     if cb is not None \
        and not self._revlist(HEAD + '...' + rev) \
        and not self.IsDirty(consider_untracked = False):
@@ -1065,17 +1068,11 @@ class Project(object):
           raise
 
   def _InitRemote(self):
-    if self.remote.fetchUrl:
+    if self.remote.url:
       remote = self.GetRemote(self.remote.name)
-
-      url = self.remote.fetchUrl
-      while url.endswith('/'):
-        url = url[:-1]
-      url += '/%s.git' % self.name
-      remote.url = url
-      remote.review = self.remote.reviewUrl
-      if remote.projectname is None:
-        remote.projectname = self.name
+      remote.url = self.remote.url
+      remote.review = self.remote.review
+      remote.projectname = self.name
 
       if self.worktree:
         remote.ResetFetch(mirror=False)
@@ -1083,37 +1080,27 @@ class Project(object):
         remote.ResetFetch(mirror=True)
       remote.Save()
 
-    for r in self.extraRemotes.values():
-      remote = self.GetRemote(r.name)
-      remote.url = r.fetchUrl
-      remote.review = r.reviewUrl
-      if r.projectName:
-        remote.projectname = r.projectName
-      elif remote.projectname is None:
-        remote.projectname = self.name
-      remote.ResetFetch()
-      remote.Save()
-
   def _InitMRef(self):
     if self.manifest.branch:
-      msg = 'manifest set to %s' % self.revision
-      ref = R_M + self.manifest.branch
-      cur = self.bare_ref.symref(ref)
-
-      if IsId(self.revision):
-        if cur != '' or self.bare_ref.get(ref) != self.revision:
-          dst = self.revision + '^0'
-          self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
-      else:
-        remote = self.GetRemote(self.remote.name)
-        dst = remote.ToLocal(self.revision)
-        if cur != dst:
-          self.bare_git.symbolic_ref('-m', msg, ref, dst)
+      self._InitAnyMRef(R_M + self.manifest.branch)
 
   def _InitMirrorHead(self):
-    dst = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    msg = 'manifest set to %s' % self.revision
-    self.bare_git.SetHead(dst, message=msg)
+    self._InitAnyMRef(HEAD)
+
+  def _InitAnyMRef(self, ref):
+    cur = self.bare_ref.symref(ref)
+
+    if self.revisionId:
+      if cur != '' or self.bare_ref.get(ref) != self.revisionId:
+        msg = 'manifest set to %s' % self.revisionId
+        dst = self.revisionId + '^0'
+        self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
+    else:
+      remote = self.GetRemote(self.remote.name)
+      dst = remote.ToLocal(self.revisionExpr)
+      if cur != dst:
+        msg = 'manifest set to %s' % self.revisionExpr
+        self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
   def _InitWorkTree(self):
     dotgit = os.path.join(self.worktree, '.git')
@@ -1140,14 +1127,11 @@ class Project(object):
           else:
             raise
 
-      rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-      rev = self.bare_git.rev_parse('%s^0' % rev)
-
-      _lwrite(os.path.join(dotgit, HEAD), '%s\n' % rev)
+      _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
       cmd = ['read-tree', '--reset', '-u']
       cmd.append('-v')
-      cmd.append('HEAD')
+      cmd.append(HEAD)
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
       self._CopyFiles()
@@ -1155,11 +1139,11 @@ class Project(object):
   def _gitdir_path(self, path):
     return os.path.join(self.gitdir, path)
 
-  def _revlist(self, *args):
-    cmd = []
-    cmd.extend(args)
-    cmd.append('--')
-    return self.work_git.rev_list(*args)
+  def _revlist(self, *args, **kw):
+    a = []
+    a.extend(args)
+    a.append('--')
+    return self.work_git.rev_list(*a, **kw)
 
   @property
   def _allrefs(self):
@@ -1284,8 +1268,11 @@ class Project(object):
       self.update_ref('-d', name, old)
       self._project.bare_ref.deleted(name)
 
-    def rev_list(self, *args):
-      cmdv = ['rev-list']
+    def rev_list(self, *args, **kw):
+      if 'format' in kw:
+        cmdv = ['log', '--pretty=format:%s' % kw['format']]
+      else:
+        cmdv = ['rev-list']
       cmdv.extend(args)
       p = GitCommand(self._project,
                      cmdv,
@@ -1294,7 +1281,9 @@ class Project(object):
                      capture_stderr = True)
       r = []
       for line in p.process.stdout:
-        r.append(line[:-1])
+        if line[-1] == '\n':
+          line = line[:-1]
+        r.append(line)
       if p.Wait() != 0:
         raise GitError('%s rev-list %s: %s' % (
                        self._project.name,
@@ -1441,9 +1430,10 @@ class MetaProject(Project):
                      name = name,
                      gitdir = gitdir,
                      worktree = worktree,
-                     remote = Remote('origin'),
+                     remote = RemoteSpec('origin'),
                      relpath = '.repo/%s' % name,
-                     revision = 'refs/heads/master')
+                     revisionExpr = 'refs/heads/master',
+                     revisionId = None)
 
   def PreSync(self):
     if self.Exists:
@@ -1451,7 +1441,8 @@ class MetaProject(Project):
       if cb:
         base = self.GetBranch(cb).merge
         if base:
-          self.revision = base
+          self.revisionExpr = base
+          self.revisionId = None
 
   @property
   def LastFetch(self):
@@ -1465,16 +1456,11 @@ class MetaProject(Project):
   def HasChanges(self):
     """Has the remote received new commits not yet checked out?
     """
-    if not self.remote or not self.revision:
+    if not self.remote or not self.revisionExpr:
       return False
 
     all = self.bare_ref.all
-    rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    if rev in all:
-      revid = all[rev]
-    else:
-      revid = rev
-
+    revid = self.GetRevisionId(all)
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       try:
@@ -1484,6 +1470,6 @@ class MetaProject(Project):
 
     if revid == head:
       return False
-    elif self._revlist(not_rev(HEAD), rev):
+    elif self._revlist(not_rev(HEAD), revid):
       return True
     return False

remote.py

@@ -1,25 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-class Remote(object):
-  def __init__(self, name,
-               fetch=None,
-               review=None,
-               projectName=None):
-    self.name = name
-    self.fetchUrl = fetch
-    self.reviewUrl = review
-    self.projectName = projectName
-    self.requiredCommits = []

repo

@@ -505,7 +505,7 @@ def _RunSelf(wrapper_path):
   my_git = os.path.join(my_dir, '.git')
 
   if os.path.isfile(my_main) and os.path.isdir(my_git):
-    for name in ['manifest.py',
+    for name in ['git_config.py',
                  'project.py',
                  'subcmds']:
       if not os.path.exists(os.path.join(my_dir, name)):

subcmds/branches.py

@@ -61,12 +61,34 @@ class Branches(Command):
 %prog [<project>...]
 
 Summarizes the currently available topic branches.
-"""
 
-  def _Options(self, p):
-    p.add_option('-a', '--all',
-                 dest='all', action='store_true',
-                 help='show all branches, not just the majority')
+Branch Display
+--------------
+
+The branch display output by this command is organized into four
+columns of information; for example:
+
+ *P nocolor                   | in repo
+    repo2                     |
+
+The first column contains a * if the branch is the currently
+checked out branch in any of the specified projects, or a blank
+if no project has the branch checked out.
+
+The second column contains either blank, p or P, depending upon
+the upload status of the branch.
+
+ (blank): branch not yet published by repo upload
+       P: all commits were published by repo upload
+       p: only some commits were published by repo upload
+
+The third column contains the branch name.
+
+The fourth column (after the | separator) lists the projects that
+the branch appears in, or does not appear in.  If no project list
+is shown, then the branch appears in all projects.
+
+"""
 
   def Execute(self, opt, args):
     projects = self.GetProjects(args)
@@ -84,18 +106,6 @@ Summarizes the currently available topic branches.
     names = all.keys()
     names.sort()
 
-    if not opt.all and not args:
-      # No -a and no specific projects listed; try to filter the
-      # results down to only the majority of projects.
-      #
-      n = []
-      for name in names:
-        i = all[name]
-        if i.IsCurrent \
-        or 80 <= (100 * len(i.projects)) / project_cnt:
-          n.append(name)
-      names = n
-
     if not names:
       print >>sys.stderr, '   (no branches)'
       return
@@ -126,7 +136,7 @@ Summarizes the currently available topic branches.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
-      if in_cnt < project_cnt and (in_cnt == 1 or opt.all):
+      if in_cnt < project_cnt and (in_cnt == 1):
         fmt = out.write
         paths = []
         if in_cnt < project_cnt - in_cnt: 

subcmds/forall.py

@@ -160,10 +160,8 @@ terminal and are not redirected.
       setenv('REPO_PROJECT', project.name)
       setenv('REPO_PATH', project.relpath)
       setenv('REPO_REMOTE', project.remote.name)
-      setenv('REPO_LREV', project\
-        .GetRemote(project.remote.name)\
-        .ToLocal(project.revision))
-      setenv('REPO_RREV', project.revision)
+      setenv('REPO_LREV', project.GetRevisionId())
+      setenv('REPO_RREV', project.revisionExpr)
 
       if mirror:
         setenv('GIT_DIR', project.gitdir)

subcmds/grep.py

@@ -17,7 +17,7 @@ import sys
 from optparse import SUPPRESS_HELP
 from color import Coloring
 from command import PagedCommand
-from git_command import GitCommand
+from git_command import git_require, GitCommand
 
 class GrepColoring(Coloring):
   def __init__(self, config):
@@ -158,7 +158,7 @@ contain a line that matches both expressions:
     out = GrepColoring(self.manifest.manifestProject.config)
 
     cmd_argv = ['grep']
-    if out.is_on:
+    if out.is_on and git_require((1,6,3)):
       cmd_argv.append('--color')
     cmd_argv.extend(getattr(opt,'cmd_argv',[]))
 

subcmds/init.py

@@ -19,9 +19,8 @@ import sys
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
-from remote import Remote
 from project import SyncBuffer
-from git_command import git, MIN_GIT_VERSION
+from git_command import git_require, MIN_GIT_VERSION
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
@@ -86,19 +85,6 @@ to update the working directory files.
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
-  def _CheckGitVersion(self):
-    ver_str = git.version()
-    if not ver_str.startswith('git version '):
-      print >>sys.stderr, 'error: "%s" unsupported' % ver_str
-      sys.exit(1)
-
-    ver_str = ver_str[len('git version '):].strip()
-    ver_act = tuple(map(lambda x: int(x), ver_str.split('.')[0:3]))
-    if ver_act < MIN_GIT_VERSION:
-      need = '.'.join(map(lambda x: str(x), MIN_GIT_VERSION))
-      print >>sys.stderr, 'fatal: git %s or later required' % need
-      sys.exit(1)
-
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
@@ -114,12 +100,12 @@ to update the working directory files.
       m._InitGitDir()
 
       if opt.manifest_branch:
-        m.revision = opt.manifest_branch
+        m.revisionExpr = opt.manifest_branch
       else:
-        m.revision = 'refs/heads/master'
+        m.revisionExpr = 'refs/heads/master'
     else:
       if opt.manifest_branch:
-        m.revision = opt.manifest_branch
+        m.revisionExpr = opt.manifest_branch
       else:
         m.PreSync()
 
@@ -215,7 +201,7 @@ to update the working directory files.
       gc.SetString('color.ui', 'auto')
 
   def Execute(self, opt, args):
-    self._CheckGitVersion()
+    git_require(MIN_GIT_VERSION, fail=True)
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 

subcmds/sync.py

@@ -16,12 +16,15 @@
 from optparse import SUPPRESS_HELP
 import os
 import re
+import shutil
 import subprocess
 import sys
 import time
 
 from git_command import GIT
 from project import HEAD
+from project import Project
+from project import RemoteSpec
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
 from project import R_HEADS
@@ -117,6 +120,61 @@ later is required to fix a server side protocol bug.
     pm.end()
     return fetched
 
+  def UpdateProjectList(self):
+    new_project_paths = []
+    for project in self.manifest.projects.values():
+      if project.relpath:
+        new_project_paths.append(project.relpath)
+    file_name = 'project.list'
+    file_path = os.path.join(self.manifest.repodir, file_name)
+    old_project_paths = []
+
+    if os.path.exists(file_path):
+      fd = open(file_path, 'r')
+      try:
+        old_project_paths = fd.read().split('\n')
+      finally:
+        fd.close()
+      for path in old_project_paths:
+        if not path:
+          continue
+        if path not in new_project_paths:
+          project = Project(
+                         manifest = self.manifest,
+                         name = path,
+                         remote = RemoteSpec('origin'),
+                         gitdir = os.path.join(self.manifest.topdir,
+                                               path, '.git'),
+                         worktree = os.path.join(self.manifest.topdir, path),
+                         relpath = path,
+                         revisionExpr = 'HEAD',
+                         revisionId = None)
+          if project.IsDirty():
+            print >>sys.stderr, 'error: Cannot remove project "%s": \
+uncommitted changes are present' % project.relpath
+            print >>sys.stderr, '       commit changes, then run sync again'
+            return -1
+          else:
+            print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
+            shutil.rmtree(project.worktree)
+            # Try deleting parent subdirs if they are empty
+            dir = os.path.dirname(project.worktree)
+            while dir != self.manifest.topdir:
+              try:
+                os.rmdir(dir)
+              except OSError:
+                break
+              dir = os.path.dirname(dir)
+
+    new_project_paths.sort()
+    fd = open(file_path, 'w')
+    try:
+      fd.write('\n'.join(new_project_paths))
+      fd.write('\n')
+    finally:
+      fd.close()
+    return 0
+
   def Execute(self, opt, args):
     if opt.network_only and opt.detach_head:
       print >>sys.stderr, 'error: cannot combine -n and -d'
@@ -164,6 +222,13 @@ later is required to fix a server side protocol bug.
             missing.append(project)
         self._Fetch(missing)
 
+    if self.manifest.IsMirror:
+      # bail out now, we have no working tree
+      return
+
+    if self.UpdateProjectList():
+      sys.exit(1)
+
     syncbuf = SyncBuffer(mp.config,
                          detach_head = opt.detach_head)
     pm = Progress('Syncing work tree', len(all))
@@ -207,17 +272,14 @@ def _VerifyTag(project):
 warning: Cannot automatically authenticate repo."""
     return True
 
-  remote = project.GetRemote(project.remote.name)
-  ref = remote.ToLocal(project.revision)
-
   try:
-    cur = project.bare_git.describe(ref)
+    cur = project.bare_git.describe(project.GetRevisionId())
   except GitError:
     cur = None
 
   if not cur \
      or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
-    rev = project.revision
+    rev = project.revisionExpr
     if rev.startswith(R_HEADS):
       rev = rev[len(R_HEADS):]