Support detached HEAD in manifest repository

If the manifest repository is on a detached HEAD and we are parsing
an XML formatted manifest we should simply set the branch property
to None, rather than crash with an AttributeError.

Signed-off-by: Shawn O. Pearce <sop@google.com>

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>repo</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/repo</path>
+</pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.4</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+</pydev_project>

color.py

@@ -110,6 +110,9 @@ class Coloring(object):
   def write(self, fmt, *args):
     self._out.write(fmt % args)
 
+  def flush(self):
+    self._out.flush()
+
   def nl(self):
     self._out.write('\n')
 

command.py

@@ -27,6 +27,9 @@ class Command(object):
   manifest = None
   _optparse = None
 
+  def WantPager(self, opt):
+    return False
+
   @property
   def OptionParser(self):
     if self._optparse is None:
@@ -109,11 +112,15 @@ class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and
      must not run within a pager, even if the user asks to.
   """
+  def WantPager(self, opt):
+    return False
 
 class PagedCommand(Command):
   """Command which defaults to output in a pager, as its
      display tends to be larger than one screen full.
   """
+  def WantPager(self, opt):
+    return True
 
 class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,

docs/manifest-format.txt

@@ -23,32 +23,23 @@ following DTD:
     <!ELEMENT manifest (remote*,
                         default?,
                         remove-project*,
-                        project*,
-                        add-remote*)>
+                        project*)>
   
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
-    <!ATTLIST remote project-name CDATA #IMPLIED>
   
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote   IDREF #IMPLIED>
     <!ATTLIST default revision CDATA #IMPLIED>
   
-    <!ELEMENT project (remote*)>
+    <!ELEMENT project (EMPTY)>
     <!ATTLIST project name     CDATA #REQUIRED>
     <!ATTLIST project path     CDATA #IMPLIED>
     <!ATTLIST project remote   IDREF #IMPLIED>
     <!ATTLIST project revision CDATA #IMPLIED>
   
-    <!ELEMENT add-remote (EMPTY)>
-    <!ATTLIST add-remote to-project   ID    #REQUIRED>
-    <!ATTLIST add-remote name         ID    #REQUIRED>
-    <!ATTLIST add-remote fetch        CDATA #REQUIRED>
-    <!ATTLIST add-remote review       CDATA #IMPLIED>
-    <!ATTLIST add-remote project-name CDATA #IMPLIED>
-  
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
   ]>
@@ -82,25 +73,6 @@ Attribute `review`: Hostname of the Gerrit server where reviews
 are uploaded to by `repo upload`.  This attribute is optional;
 if not specified then `repo upload` will not function.
 
-Attribute `project-name`: Specifies the name of this project used
-by the review server given in the review attribute of this element.
-Only permitted when the remote element is nested inside of a project
-element (see below).  If not given, defaults to the name supplied
-in the project's name attribute.
-
-Element add-remote
-------------------
-
-Adds a remote to an existing project, whose name is given by the
-to-project attribute.  This is functionally equivalent to nesting
-a remote element under the project, but has the advantage that it
-can be specified in the uesr's `local_manifest.xml` to add a remote
-to a project declared by the normal manifest.
-
-The element can be used to add a fork of an existing project that
-the user needs to work with.
-
-
 Element default
 ---------------
 
@@ -152,13 +124,6 @@ Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
 the default element is used.
 
-Child element `remote`: Described like the top-level remote element,
-but adds an additional remote to only this project.  These additional
-remotes are fetched from first on the initial `repo sync`, causing
-the majority of the project's object database to be obtained through
-these additional remotes.
-
-
 Element remove-project
 ----------------------
 
@@ -191,8 +156,3 @@ For example:
 Users may add projects to the local manifest prior to a `repo sync`
 invocation, instructing repo to automatically download and manage
 these extra projects.
-
-Currently the only supported feature of a local manifest is to
-add new remotes and/or projects.  In the future a local manifest
-may support picking different revisions of a project, or deleting
-projects specified in the default manifest.

editor.py

@@ -76,9 +76,20 @@ least one of these before using this command."""
       os.close(fd)
       fd = None
 
-      if subprocess.Popen(editor + [path]).wait() != 0:
-        raise EditorError()
-      return open(path).read()
+      try:
+        rc = subprocess.Popen(editor + [path]).wait()
+      except OSError, e:
+        raise EditorError('editor failed, %s: %s %s'
+          % (str(e), cls._GetEditor(), path))
+      if rc != 0:
+        raise EditorError('editor failed with exit status %d: %s %s'
+          % (rc, cls._GetEditor(), path))
+
+      fd2 = open(path)
+      try:
+        return fd2.read()
+      finally:
+        fd2.close()
     finally:
       if fd:
         os.close(fd)

error.py

@@ -24,6 +24,11 @@ class ManifestInvalidRevisionError(Exception):
 class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """
+  def __init__(self, reason):
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
 
 class GitError(Exception):
   """Unspecified internal error from git.

git_command.py

@@ -16,19 +16,40 @@
 import os
 import sys
 import subprocess
+import tempfile
 from error import GitError
+from trace import REPO_TRACE, IsTrace, Trace
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
 GIT_DIR = 'GIT_DIR'
-REPO_TRACE = 'REPO_TRACE'
 
 LAST_GITDIR = None
 LAST_CWD = None
-try:
-  TRACE = os.environ[REPO_TRACE] == '1'
-except KeyError:
-  TRACE = False
+
+_ssh_proxy_path = None
+_ssh_sock_path = None
+
+def _ssh_sock(create=True):
+  global _ssh_sock_path
+  if _ssh_sock_path is None:
+    if not create:
+      return None
+    dir = '/tmp'
+    if not os.path.exists(dir):
+      dir = tempfile.gettempdir()
+    _ssh_sock_path = os.path.join(
+      tempfile.mkdtemp('', 'ssh-', dir),
+      'master-%r@%h:%p')
+  return _ssh_sock_path
+
+def _ssh_proxy():
+  global _ssh_proxy_path
+  if _ssh_proxy_path is None:
+    _ssh_proxy_path = os.path.join(
+      os.path.dirname(__file__),
+      'git_ssh')
+  return _ssh_proxy_path
 
 
 class _GitCall(object):
@@ -47,6 +68,30 @@ class _GitCall(object):
     return fun
 git = _GitCall()
 
+_git_version = None
+
+def git_require(min_version, fail=False):
+  global _git_version
+
+  if _git_version is None:
+    ver_str = git.version()
+    if ver_str.startswith('git version '):
+      _git_version = tuple(
+        map(lambda x: int(x),
+          ver_str[len('git version '):].strip().split('.')[0:3]
+        ))
+    else:
+      print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
+      sys.exit(1)
+
+  if min_version <= _git_version:
+    return True
+  if fail:
+    need = '.'.join(map(lambda x: str(x), min_version))
+    print >>sys.stderr, 'fatal: git %s or later required' % need
+    sys.exit(1)
+  return False
+
 class GitCommand(object):
   def __init__(self,
                project,
@@ -56,6 +101,7 @@ class GitCommand(object):
                capture_stdout = False,
                capture_stderr = False,
                disable_editor = False,
+               ssh_proxy = False,
                cwd = None,
                gitdir = None):
     env = dict(os.environ)
@@ -72,6 +118,9 @@ class GitCommand(object):
 
     if disable_editor:
       env['GIT_EDITOR'] = ':'
+    if ssh_proxy:
+      env['REPO_SSH_SOCK'] = _ssh_sock()
+      env['GIT_SSH'] = _ssh_proxy()
 
     if project:
       if not cwd:
@@ -101,7 +150,7 @@ class GitCommand(object):
     else:
       stderr = None
 
-    if TRACE:
+    if IsTrace():
       global LAST_CWD
       global LAST_GITDIR
 
@@ -127,7 +176,7 @@ class GitCommand(object):
         dbg += ' 1>|'
       if stderr == subprocess.PIPE:
         dbg += ' 2>|'
-      print >>sys.stderr, dbg
+      Trace('%s', dbg)
 
     try:
       p = subprocess.Popen(command,

git_config.py

@@ -13,12 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import cPickle
 import os
 import re
+import subprocess
 import sys
+import time
+from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
-from git_command import GitCommand
+from trace import Trace
+from git_command import GitCommand, _ssh_sock
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
@@ -29,6 +34,13 @@ REVIEW_CACHE = dict()
 def IsId(rev):
   return ID_RE.match(rev)
 
+def _key(name):
+  parts = name.split('.')
+  if len(parts) < 2:
+    return name.lower()
+  parts[ 0] = parts[ 0].lower()
+  parts[-1] = parts[-1].lower()
+  return '.'.join(parts)
 
 class GitConfig(object):
   _ForUser = None
@@ -44,18 +56,25 @@ class GitConfig(object):
     return cls(file = os.path.join(gitdir, 'config'),
                defaults = defaults)
 
-  def __init__(self, file, defaults=None):
+  def __init__(self, file, defaults=None, pickleFile=None):
     self.file = file
     self.defaults = defaults
     self._cache_dict = None
+    self._section_dict = None
     self._remotes = {}
     self._branches = {}
 
+    if pickleFile is None:
+      self._pickle = os.path.join(
+        os.path.dirname(self.file),
+        '.repopickle_' + os.path.basename(self.file))
+    else:
+      self._pickle = pickleFile
+
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
     """
-    name = name.lower()
-    if name in self._cache:
+    if _key(name) in self._cache:
       return True
     if include_defaults and self.defaults:
       return self.defaults.Has(name, include_defaults = True)
@@ -83,10 +102,8 @@ class GitConfig(object):
        This configuration file is used first, if the key is not
        defined or all = True then the defaults are also searched.
     """
-    name = name.lower()
-
     try:
-      v = self._cache[name]
+      v = self._cache[_key(name)]
     except KeyError:
       if self.defaults:
         return self.defaults.GetString(name, all = all)
@@ -110,16 +127,16 @@ class GitConfig(object):
        The supplied value should be either a string,
        or a list of strings (to store multiple values).
     """
-    name = name.lower()
+    key = _key(name)
 
     try:
-      old = self._cache[name]
+      old = self._cache[key]
     except KeyError:
       old = []
 
     if value is None:
       if old:
-        del self._cache[name]
+        del self._cache[key]
         self._do('--unset-all', name)
 
     elif isinstance(value, list):
@@ -130,13 +147,13 @@ class GitConfig(object):
         self.SetString(name, value[0])
 
       elif old != value:
-        self._cache[name] = list(value)
+        self._cache[key] = list(value)
         self._do('--replace-all', name, value[0])
         for i in xrange(1, len(value)):
           self._do('--add', name, value[i])
 
     elif len(old) != 1 or old[0] != value:
-      self._cache[name] = [value]
+      self._cache[key] = [value]
       self._do('--replace-all', name, value)
 
   def GetRemote(self, name):
@@ -159,6 +176,38 @@ class GitConfig(object):
       self._branches[b.name] = b
     return b
 
+  def GetSubSections(self, section):
+    """List all subsection names matching $section.*.*
+    """
+    return self._sections.get(section, set())
+
+  def HasSection(self, section, subsection = ''):
+    """Does at least one key in section.subsection exist?
+    """
+    try:
+      return subsection in self._sections[section]
+    except KeyError:
+      return False
+
+  @property
+  def _sections(self):
+    d = self._section_dict
+    if d is None:
+      d = {}
+      for name in self._cache.keys():
+        p = name.split('.')
+        if 2 == len(p):
+          section = p[0]
+          subsect = ''
+        else:
+          section = p[0]
+          subsect = '.'.join(p[1:-1])
+        if section not in d:
+          d[section] = set()
+        d[section].add(subsect)
+        self._section_dict = d
+    return d
+
   @property
   def _cache(self):
     if self._cache_dict is None:
@@ -166,13 +215,57 @@ class GitConfig(object):
     return self._cache_dict
 
   def _Read(self):
+    d = self._ReadPickle()
+    if d is None:
+      d = self._ReadGit()
+      self._SavePickle(d)
+    return d
+
+  def _ReadPickle(self):
+    try:
+      if os.path.getmtime(self._pickle) \
+      <= os.path.getmtime(self.file):
+        os.remove(self._pickle)
+        return None
+    except OSError:
+      return None
+    try:
+      Trace(': unpickle %s', self.file)
+      fd = open(self._pickle, 'rb')
+      try:
+        return cPickle.load(fd)
+      finally:
+        fd.close()
+    except EOFError:
+      os.remove(self._pickle)
+      return None
+    except IOError:
+      os.remove(self._pickle)
+      return None
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+      return None
+
+  def _SavePickle(self, cache):
+    try:
+      fd = open(self._pickle, 'wb')
+      try:
+        cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
+      finally:
+        fd.close()
+    except IOError:
+      os.remove(self._pickle)
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+
+  def _ReadGit(self):
     d = self._do('--null', '--list')
     c = {}
     while d:
       lf = d.index('\n')
       nul = d.index('\0', lf + 1)
 
-      key = d[0:lf]
+      key = _key(d[0:lf])
       val = d[lf + 1:nul]
 
       if key in c:
@@ -253,6 +346,82 @@ class RefSpec(object):
     return s
 
 
+_ssh_cache = {}
+_ssh_master = True
+
+def _open_ssh(host, port):
+  global _ssh_master
+
+  key = '%s:%s' % (host, port)
+  if key in _ssh_cache:
+    return True
+
+  if not _ssh_master \
+  or 'GIT_SSH' in os.environ \
+  or sys.platform in ('win32', 'cygwin'):
+    # failed earlier, or cygwin ssh can't do this
+    #
+    return False
+
+  command = ['ssh',
+             '-o','ControlPath %s' % _ssh_sock(),
+             '-p',str(port),
+             '-M',
+             '-N',
+             host]
+  try:
+    Trace(': %s', ' '.join(command))
+    p = subprocess.Popen(command)
+  except Exception, e:
+    _ssh_master = False
+    print >>sys.stderr, \
+      '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+      % (host,port, str(e))
+    return False
+
+  _ssh_cache[key] = p
+  time.sleep(1)
+  return True
+
+def close_ssh():
+  for key,p in _ssh_cache.iteritems():
+    try:
+      os.kill(p.pid, SIGTERM)
+      p.wait()
+    except OSError:
+      pass
+  _ssh_cache.clear()
+
+  d = _ssh_sock(create=False)
+  if d:
+    try:
+      os.rmdir(os.path.dirname(d))
+    except OSError:
+      pass
+
+URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
+URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
+
+def _preconnect(url):
+  m = URI_ALL.match(url)
+  if m:
+    scheme = m.group(1)
+    host = m.group(2)
+    if ':' in host:
+      host, port = host.split(':')
+    else:
+      port = 22
+    if scheme in ('ssh', 'git+ssh', 'ssh+git'):
+      return _open_ssh(host, port)
+    return False
+
+  m = URI_SCP.match(url)
+  if m:
+    host = m.group(1)
+    return _open_ssh(host, 22)
+
+  return False
+
 class Remote(object):
   """Configuration options related to a remote.
   """
@@ -266,6 +435,9 @@ class Remote(object):
                      self._Get('fetch', all=True))
     self._review_protocol = None
 
+  def PreConnectFetch(self):
+    return _preconnect(self.url)
+
   @property
   def ReviewProtocol(self):
     if self._review_protocol is None:
@@ -399,11 +571,23 @@ class Branch(object):
   def Save(self):
     """Save this branch back into the configuration.
     """
-    self._Set('merge', self.merge)
-    if self.remote:
-      self._Set('remote', self.remote.name)
+    if self._config.HasSection('branch', self.name):
+      if self.remote:
+        self._Set('remote', self.remote.name)
+      else:
+        self._Set('remote', None)
+      self._Set('merge', self.merge)
+
     else:
-      self._Set('remote', None)
+      fd = open(self._config.file, 'ab')
+      try:
+        fd.write('[branch "%s"]\n' % self.name)
+        if self.remote:
+          fd.write('\tremote = %s\n' % self.remote.name)
+        if self.merge:
+          fd.write('\tmerge = %s\n' % self.merge)
+      finally:
+        fd.close()
 
   def _Set(self, key, value):
     key = 'branch.%s.%s' % (self.name, key)

git_refs.py

@@ -0,0 +1,160 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from trace import Trace
+
+HEAD    = 'HEAD'
+R_HEADS = 'refs/heads/'
+R_TAGS  = 'refs/tags/'
+R_PUB   = 'refs/published/'
+R_M     = 'refs/remotes/m/'
+
+
+class GitRefs(object):
+  def __init__(self, gitdir):
+    self._gitdir = gitdir
+    self._phyref = None
+    self._symref = None
+    self._mtime = {}
+
+  @property
+  def all(self):
+    self._EnsureLoaded()
+    return self._phyref
+
+  def get(self, name):
+    try:
+      return self.all[name]
+    except KeyError:
+      return ''
+
+  def deleted(self, name):
+    if self._phyref is not None:
+      if name in self._phyref:
+        del self._phyref[name]
+
+      if name in self._symref:
+        del self._symref[name]
+
+      if name in self._mtime:
+        del self._mtime[name]
+
+  def symref(self, name):
+    try:
+      self._EnsureLoaded()
+      return self._symref[name]
+    except KeyError:
+      return ''
+
+  def _EnsureLoaded(self):
+    if self._phyref is None or self._NeedUpdate():
+      self._LoadAll()
+
+  def _NeedUpdate(self):
+    Trace(': scan refs %s', self._gitdir)
+
+    for name, mtime in self._mtime.iteritems():
+      try:
+        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+          return True
+      except OSError:
+        return True
+    return False
+
+  def _LoadAll(self):
+    Trace(': load refs %s', self._gitdir)
+
+    self._phyref = {}
+    self._symref = {}
+    self._mtime = {}
+
+    self._ReadPackedRefs()
+    self._ReadLoose('refs/')
+    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+
+    scan = self._symref
+    attempts = 0
+    while scan and attempts < 5:
+      scan_next = {}
+      for name, dest in scan.iteritems():
+        if dest in self._phyref:
+          self._phyref[name] = self._phyref[dest]
+        else:
+          scan_next[name] = dest
+      scan = scan_next
+      attempts += 1
+
+  def _ReadPackedRefs(self):
+    path = os.path.join(self._gitdir, 'packed-refs')
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except IOError:
+      return
+    except OSError:
+      return
+    try:
+      for line in fd:
+        if line[0] == '#':
+          continue
+        if line[0] == '^':
+          continue
+
+        line = line[:-1]
+        p = line.split(' ')
+        id = p[0]
+        name = p[1]
+
+        self._phyref[name] = id
+    finally:
+      fd.close()
+    self._mtime['packed-refs'] = mtime
+
+  def _ReadLoose(self, prefix):
+    base = os.path.join(self._gitdir, prefix)
+    for name in os.listdir(base):
+      p = os.path.join(base, name)
+      if os.path.isdir(p):
+        self._mtime[prefix] = os.path.getmtime(base)
+        self._ReadLoose(prefix + name + '/')
+      elif name.endswith('.lock'):
+        pass
+      else:
+        self._ReadLoose1(p, prefix + name)
+
+  def _ReadLoose1(self, path, name):
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except OSError:
+      return
+    except IOError:
+      return
+    try:
+      id = fd.readline()
+    finally:
+      fd.close()
+
+    if not id:
+      return
+    id = id[:-1]
+
+    if id.startswith('ref: '):
+      self._symref[name] = id[5:]
+    else:
+      self._phyref[name] = id
+    self._mtime[name] = mtime

git_ssh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec ssh -o "ControlPath $REPO_SSH_SOCK" "$@"

main.py

@@ -27,7 +27,8 @@ import os
 import re
 import sys
 
-import git_command
+from trace import SetTrace
+from git_config import close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
@@ -35,7 +36,7 @@ from editor import Editor
 from error import ManifestInvalidRevisionError
 from error import NoSuchProjectError
 from error import RepoChangedException
-from manifest import Manifest
+from manifest_xml import XmlManifest
 from pager import RunPager
 
 from subcmds import all as all_commands
@@ -79,7 +80,7 @@ class _Repo(object):
     gopts, gargs = global_options.parse_args(glob)
 
     if gopts.trace:
-      git_command.TRACE = True
+      SetTrace()
     if gopts.show_version:
       if name == 'help':
         name = 'version'
@@ -96,7 +97,7 @@ class _Repo(object):
       sys.exit(1)
 
     cmd.repodir = self.repodir
-    cmd.manifest = Manifest(cmd.repodir)
+    cmd.manifest = XmlManifest(cmd.repodir)
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -105,6 +106,8 @@ class _Repo(object):
             % name
       sys.exit(1)
 
+    copts, cargs = cmd.OptionParser.parse_args(argv)
+
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
       if gopts.pager:
@@ -112,11 +115,10 @@ class _Repo(object):
       else:
         use_pager = config.GetBoolean('pager.%s' % name)
         if use_pager is None:
-          use_pager = isinstance(cmd, PagedCommand)
+          use_pager = cmd.WantPager(copts)
       if use_pager:
         RunPager(config)
 
-    copts, cargs = cmd.OptionParser.parse_args(argv)
     try:
       cmd.Execute(copts, cargs)
     except ManifestInvalidRevisionError, e:
@@ -211,7 +213,10 @@ def _Main(argv):
 
   repo = _Repo(opt.repodir)
   try:
-    repo._Run(argv)
+    try:
+      repo._Run(argv)
+    finally:
+      close_ssh()
   except KeyboardInterrupt:
     sys.exit(1)
   except RepoChangedException, rce:

manifest_xml.py

@@ -18,8 +18,7 @@ import sys
 import xml.dom.minidom
 
 from git_config import GitConfig, IsId
-from project import Project, MetaProject, R_HEADS, HEAD
-from remote import Remote
+from project import RemoteSpec, Project, MetaProject, R_HEADS, HEAD
 from error import ManifestParseError
 
 MANIFEST_FILE_NAME = 'manifest.xml'
@@ -28,11 +27,26 @@ LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 class _Default(object):
   """Project defaults within the manifest."""
 
-  revision = None
+  revisionExpr = None
   remote = None
 
+class _XmlRemote(object):
+  def __init__(self,
+               name,
+               fetch=None,
+               review=None):
+    self.name = name
+    self.fetchUrl = fetch
+    self.reviewUrl = review
 
-class Manifest(object):
+  def ToRemoteSpec(self, projectName):
+    url = self.fetchUrl
+    while url.endswith('/'):
+      url = url[:-1]
+    url += '/%s.git' % projectName
+    return RemoteSpec(self.name, url, self.reviewUrl)
+
+class XmlManifest(object):
   """manages the repo configuration file"""
 
   def __init__(self, repodir):
@@ -80,8 +94,6 @@ class Manifest(object):
     e.setAttribute('fetch', r.fetchUrl)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
-    if r.projectName is not None:
-      e.setAttribute('project-name', r.projectName)
 
   def Save(self, fd, peg_rev=False):
     """Write the current manifest out to the given file descriptor.
@@ -104,9 +116,9 @@ class Manifest(object):
     if d.remote:
       have_default = True
       e.setAttribute('remote', d.remote.name)
-    if d.revision:
+    if d.revisionExpr:
       have_default = True
-      e.setAttribute('revision', d.revision)
+      e.setAttribute('revision', d.revisionExpr)
     if have_default:
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
@@ -126,15 +138,13 @@ class Manifest(object):
       if peg_rev:
         if self.IsMirror:
           e.setAttribute('revision',
-                         p.bare_git.rev_parse(p.revision + '^0'))
+                         p.bare_git.rev_parse(p.revisionExpr + '^0'))
         else:
           e.setAttribute('revision',
                          p.work_git.rev_parse(HEAD + '^0'))
-      elif not d.revision or p.revision != d.revision:
-        e.setAttribute('revision', p.revision)
+      elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
+        e.setAttribute('revision', p.revisionExpr)
 
-      for r in p.extraRemotes:
-        self._RemoteToXml(p.extraRemotes[r], doc, e)
       for c in p.copyfiles:
         ce = doc.createElement('copyfile')
         ce.setAttribute('src', c.src)
@@ -173,7 +183,7 @@ class Manifest(object):
     if not self._loaded:
       m = self.manifestProject
       b = m.GetBranch(m.CurrentBranch).merge
-      if b.startswith(R_HEADS):
+      if b is not None and b.startswith(R_HEADS):
         b = b[len(R_HEADS):]
       self.branch = b
 
@@ -245,16 +255,6 @@ class Manifest(object):
                 (project.name, self.manifestFile)
         self._projects[project.name] = project
 
-    for node in config.childNodes:
-      if node.nodeName == 'add-remote':
-        pn = self._reqatt(node, 'to-project')
-        project = self._projects.get(pn)
-        if not project:
-          raise ManifestParseError, \
-                'project %s not defined in %s' % \
-                (pn, self.manifestFile)
-        self._ParseProjectExtraRemote(project, node)
-
   def _AddMetaProjectMirror(self, m):
     name = None
     m_url = m.GetRemote(m.remote.name).url
@@ -271,7 +271,7 @@ class Manifest(object):
 
     if name is None:
       s = m_url.rindex('/') + 1
-      remote = Remote('origin', fetch = m_url[:s])
+      remote = _XmlRemote('origin', m_url[:s])
       name = m_url[s:]
 
     if name.endswith('.git'):
@@ -282,11 +282,12 @@ class Manifest(object):
       gitdir = os.path.join(self.topdir, '%s.git' % name)
       project = Project(manifest = self,
                         name = name,
-                        remote = remote,
+                        remote = remote.ToRemoteSpec(name),
                         gitdir = gitdir,
                         worktree = None,
                         relpath = None,
-                        revision = m.revision)
+                        revisionExpr = m.revisionExpr,
+                        revisionId = None)
       self._projects[project.name] = project
 
   def _ParseRemote(self, node):
@@ -298,21 +299,7 @@ class Manifest(object):
     review = node.getAttribute('review')
     if review == '':
       review = None
-
-    projectName = node.getAttribute('project-name')
-    if projectName == '':
-      projectName = None
-
-    r = Remote(name=name,
-               fetch=fetch,
-               review=review,
-               projectName=projectName)
-
-    for n in node.childNodes:
-      if n.nodeName == 'require':
-        r.requiredCommits.append(self._reqatt(n, 'commit'))
-
-    return r
+    return _XmlRemote(name, fetch, review)
 
   def _ParseDefault(self, node):
     """
@@ -320,9 +307,9 @@ class Manifest(object):
     """
     d = _Default()
     d.remote = self._get_remote(node)
-    d.revision = node.getAttribute('revision')
-    if d.revision == '':
-      d.revision = None
+    d.revisionExpr = node.getAttribute('revision')
+    if d.revisionExpr == '':
+      d.revisionExpr = None
     return d
 
   def _ParseProject(self, node):
@@ -339,10 +326,10 @@ class Manifest(object):
             "no remote for project %s within %s" % \
             (name, self.manifestFile)
 
-    revision = node.getAttribute('revision')
-    if not revision:
-      revision = self._default.revision
-    if not revision:
+    revisionExpr = node.getAttribute('revision')
+    if not revisionExpr:
+      revisionExpr = self._default.revisionExpr
+    if not revisionExpr:
       raise ManifestParseError, \
             "no revision for project %s within %s" % \
             (name, self.manifestFile)
@@ -365,29 +352,19 @@ class Manifest(object):
 
     project = Project(manifest = self,
                       name = name,
-                      remote = remote,
+                      remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
                       worktree = worktree,
                       relpath = path,
-                      revision = revision)
+                      revisionExpr = revisionExpr,
+                      revisionId = None)
 
     for n in node.childNodes:
-      if n.nodeName == 'remote':
-        self._ParseProjectExtraRemote(project, n)
-      elif n.nodeName == 'copyfile':
+      if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
 
     return project
 
-  def _ParseProjectExtraRemote(self, project, n):
-    r = self._ParseRemote(n)
-    if project.extraRemotes.get(r.name) \
-       or project.remote.name == r.name:
-      raise ManifestParseError, \
-            'duplicate remote %s in project %s in %s' % \
-            (r.name, project.name, self.manifestFile)
-    project.extraRemotes[r.name] = r
-
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
     dest = self._reqatt(node, 'dest')

progress.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 import sys
+from time import time
+from trace import IsTrace
 
 class Progress(object):
   def __init__(self, title, total=0):
@@ -21,10 +23,21 @@ class Progress(object):
     self._total = total
     self._done = 0
     self._lastp = -1
+    self._start = time()
+    self._show = False
 
   def update(self, inc=1):
     self._done += inc
 
+    if IsTrace():
+      return
+
+    if not self._show:
+      if 0.5 <= time() - self._start:
+        self._show = True
+      else:
+        return
+
     if self._total <= 0:
       sys.stderr.write('\r%s: %d, ' % (
         self._title,
@@ -43,6 +56,9 @@ class Progress(object):
         sys.stderr.flush()
 
   def end(self):
+    if IsTrace() or not self._show:
+      return
+
     if self._total <= 0:
       sys.stderr.write('\r%s: %d, done.  \n' % (
         self._title,

project.py

@@ -26,13 +26,23 @@ from git_command import GitCommand
 from git_config import GitConfig, IsId
 from error import GitError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
-from remote import Remote
 
-HEAD    = 'HEAD'
-R_HEADS = 'refs/heads/'
-R_TAGS  = 'refs/tags/'
-R_PUB   = 'refs/published/'
-R_M     = 'refs/remotes/m/'
+from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
+
+def _lwrite(path, content):
+  lock = '%s.lock' % path
+
+  fd = open(lock, 'wb')
+  try:
+    fd.write(content)
+  finally:
+    fd.close()
+
+  try:
+    os.rename(lock, path)
+  except OSError:
+    os.remove(lock)
+    raise
 
 def _error(fmt, *args):
   msg = fmt % args
@@ -144,16 +154,19 @@ class ReviewableBranch(object):
                                  self.replace_changes,
                                  people)
 
-  @property
-  def tip_url(self):
-    me = self.project.GetBranch(self.name)
-    commit = self.project.bare_git.rev_parse(R_HEADS + self.name)
-    return 'http://%s/r/%s' % (me.remote.review, commit[0:12])
-
-  @property
-  def owner_email(self):
-    return self.project.UserEmail
+  def GetPublishedRefs(self):
+    refs = {}
+    output = self.project.bare_git.ls_remote(
+      self.branch.remote.SshReviewUrl(self.project.UserEmail),
+      'refs/changes/*')
+    for line in output.split('\n'):
+      try:
+        (sha, ref) = line.split()
+        refs[sha] = ref
+      except ValueError:
+        pass
 
+    return refs
 
 class StatusColoring(Coloring):
   def __init__(self, config):
@@ -161,6 +174,7 @@ class StatusColoring(Coloring):
     self.project   = self.printer('header',    attr = 'bold')
     self.branch    = self.printer('header',    attr = 'bold')
     self.nobranch  = self.printer('nobranch',  fg = 'red')
+    self.important = self.printer('important', fg = 'red')
 
     self.added     = self.printer('added',     fg = 'green')
     self.changed   = self.printer('changed',   fg = 'red')
@@ -197,6 +211,14 @@ class _CopyFile:
       except IOError:
         _error('Cannot copy file %s to %s', src, dest)
 
+class RemoteSpec(object):
+  def __init__(self,
+               name,
+               url = None,
+               review = None):
+    self.name = name
+    self.url = url
+    self.review = review
 
 class Project(object):
   def __init__(self,
@@ -206,16 +228,24 @@ class Project(object):
                gitdir,
                worktree,
                relpath,
-               revision):
+               revisionExpr,
+               revisionId):
     self.manifest = manifest
     self.name = name
     self.remote = remote
     self.gitdir = gitdir
     self.worktree = worktree
     self.relpath = relpath
-    self.revision = revision
+    self.revisionExpr = revisionExpr
+
+    if   revisionId is None \
+     and revisionExpr \
+     and IsId(revisionExpr):
+      self.revisionId = revisionExpr
+    else:
+      self.revisionId = revisionId
+
     self.snapshots = {}
-    self.extraRemotes = {}
     self.copyfiles = []
     self.config = GitConfig.ForRepository(
                     gitdir = self.gitdir,
@@ -226,6 +256,7 @@ class Project(object):
     else:
       self.work_git = None
     self.bare_git = self._GitGetByExec(self, bare=True)
+    self.bare_ref = GitRefs(gitdir)
 
   @property
   def Exists(self):
@@ -237,14 +268,18 @@ class Project(object):
        The branch name omits the 'refs/heads/' prefix.
        None is returned if the project is on a detached HEAD.
     """
-    try:
-      b = self.work_git.GetHead()
-    except GitError:
-      return None
+    b = self.work_git.GetHead()
     if b.startswith(R_HEADS):
       return b[len(R_HEADS):]
     return None
 
+  def IsRebaseInProgress(self):
+    w = self.worktree
+    g = os.path.join(w, '.git')
+    return os.path.exists(os.path.join(g, 'rebase-apply')) \
+        or os.path.exists(os.path.join(g, 'rebase-merge')) \
+        or os.path.exists(os.path.join(w, '.dotest'))
+    
   def IsDirty(self, consider_untracked=True):
     """Is the working directory modified in some way?
     """
@@ -304,7 +339,7 @@ class Project(object):
     """Get all existing local branches.
     """
     current = self.CurrentBranch
-    all = self.bare_git.ListRefs()
+    all = self._allrefs
     heads = {}
     pubd = {}
 
@@ -342,10 +377,11 @@ class Project(object):
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
+    rb = self.IsRebaseInProgress()
     di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
-    if not di and not df and not do:
+    if not rb and not di and not df and not do:
       return 'CLEAN'
 
     out = StatusColoring(self.config)
@@ -358,6 +394,10 @@ class Project(object):
       out.branch('branch %s', branch)
     out.nl()
 
+    if rb:
+      out.important('prior sync failed; rebase still in progress')
+      out.nl()
+
     paths = list()
     paths.extend(di.keys())
     paths.extend(df.keys())
@@ -422,22 +462,31 @@ class Project(object):
 
 ## Publish / Upload ##
 
-  def WasPublished(self, branch):
+  def WasPublished(self, branch, all=None):
     """Was the branch published (uploaded) for code review?
        If so, returns the SHA-1 hash of the last published
        state for the branch.
     """
-    try:
-      return self.bare_git.rev_parse(R_PUB + branch)
-    except GitError:
-      return None
+    key = R_PUB + branch
+    if all is None:
+      try:
+        return self.bare_git.rev_parse(key)
+      except GitError:
+        return None
+    else:
+      try:
+        return all[key]
+      except KeyError:
+        return None
 
-  def CleanPublishedCache(self):
+  def CleanPublishedCache(self, all=None):
     """Prunes any stale published refs.
     """
+    if all is None:
+      all = self._allrefs
     heads = set()
     canrm = {}
-    for name, id in self._allrefs.iteritems():
+    for name, id in all.iteritems():
       if name.startswith(R_HEADS):
         heads.add(name)
       elif name.startswith(R_PUB):
@@ -545,9 +594,6 @@ class Project(object):
       self._InitGitDir()
 
     self._InitRemote()
-    for r in self.extraRemotes.values():
-      if not self._RemoteFetch(r.name):
-        return False
     if not self._RemoteFetch():
       return False
 
@@ -568,47 +614,74 @@ class Project(object):
     for file in self.copyfiles:
       file._Copy()
 
+  def GetRevisionId(self, all=None):
+    if self.revisionId:
+      return self.revisionId
+
+    rem = self.GetRemote(self.remote.name)
+    rev = rem.ToLocal(self.revisionExpr)
+
+    if all is not None and rev in all:
+      return all[rev]
+
+    try:
+      return self.bare_git.rev_parse('--verify', '%s^0' % rev)
+    except GitError:
+      raise ManifestInvalidRevisionError(
+        'revision %s in %s not found' % (self.revisionExpr,
+                                         self.name))
+
   def Sync_LocalHalf(self, syncbuf):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
     self._InitWorkTree()
-    self.CleanPublishedCache()
+    all = self.bare_ref.all
+    self.CleanPublishedCache(all)
 
-    rem = self.GetRemote(self.remote.name)
-    rev = rem.ToLocal(self.revision)
-    try:
-      self.bare_git.rev_parse('--verify', '%s^0' % rev)
-    except GitError:
-      raise ManifestInvalidRevisionError(
-        'revision %s in %s not found' % (self.revision, self.name))
-
-    branch = self.CurrentBranch
+    revid = self.GetRevisionId(all)
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      branch = head[len(R_HEADS):]
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+    else:
+      branch = None
 
     if branch is None or syncbuf.detach_head:
       # Currently on a detached HEAD.  The user is assumed to
       # not have any local modifications worth worrying about.
       #
-      if os.path.exists(os.path.join(self.worktree, '.dotest')) \
-      or os.path.exists(os.path.join(self.worktree, '.git', 'rebase-apply')):
+      if self.IsRebaseInProgress():
         syncbuf.fail(self, _PriorSyncFailedError())
         return
 
-      lost = self._revlist(not_rev(rev), HEAD)
+      if head == revid:
+        # No changes; don't do anything further.
+        #
+        return
+
+      lost = self._revlist(not_rev(revid), HEAD)
       if lost:
         syncbuf.info(self, "discarding %d commits", len(lost))
       try:
-        self._Checkout(rev, quiet=True)
+        self._Checkout(revid, quiet=True)
       except GitError, e:
         syncbuf.fail(self, e)
         return
       self._CopyFiles()
       return
 
-    branch = self.GetBranch(branch)
-    merge = branch.LocalMerge
+    if head == revid:
+      # No changes; don't do anything further.
+      #
+      return
 
-    if not merge:
+    branch = self.GetBranch(branch)
+
+    if not branch.LocalMerge:
       # The current branch has no tracking configuration.
       # Jump off it to a deatched HEAD.
       #
@@ -616,17 +689,17 @@ class Project(object):
                    "leaving %s; does not track upstream",
                    branch.name)
       try:
-        self._Checkout(rev, quiet=True)
+        self._Checkout(revid, quiet=True)
       except GitError, e:
         syncbuf.fail(self, e)
         return
       self._CopyFiles()
       return
 
-    upstream_gain = self._revlist(not_rev(HEAD), rev)
-    pub = self.WasPublished(branch.name)
+    upstream_gain = self._revlist(not_rev(HEAD), revid)
+    pub = self.WasPublished(branch.name, all)
     if pub:
-      not_merged = self._revlist(not_rev(rev), pub)
+      not_merged = self._revlist(not_rev(revid), pub)
       if not_merged:
         if upstream_gain:
           # The user has published this branch and some of those
@@ -637,79 +710,76 @@ class Project(object):
                        "branch %s is published but is now %d commits behind",
                        branch.name,
                        len(upstream_gain))
-          syncbuf.info(self, "consider merging or rebasing the unpublished commits")
         return
-      elif upstream_gain:
-        # We can fast-forward safely.
+      elif pub == head:
+        # All published commits are merged, and thus we are a
+        # strict subset.  We can fast-forward safely.
         #
         def _doff():
-          self._FastForward(rev)
+          self._FastForward(revid)
           self._CopyFiles()
         syncbuf.later1(self, _doff)
         return
-      else:
-        # Trivially no changes in the upstream.
-        #
-        return
 
-    if merge == rev:
-      try:
-        old_merge = self.bare_git.rev_parse('%s@{1}' % merge)
-      except GitError:
-        old_merge = merge
-      if old_merge == '0000000000000000000000000000000000000000' \
-         or old_merge == '':
-        old_merge = merge
-    else:
-      # The upstream switched on us.  Time to cross our fingers
-      # and pray that the old upstream also wasn't in the habit
-      # of rebasing itself.
-      #
-      syncbuf.info(self, "manifest switched %s...%s", merge, rev)
-      old_merge = merge
+    # Examine the local commits not in the remote.  Find the
+    # last one attributed to this user, if any.
+    #
+    local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
+    last_mine = None
+    cnt_mine = 0
+    for commit in local_changes:
+      commit_id, committer_email = commit.split(' ', 2)
+      if committer_email == self.UserEmail:
+        last_mine = commit_id
+        cnt_mine += 1
 
-    if rev == old_merge:
-      upstream_lost = []
-    else:
-      upstream_lost = self._revlist(not_rev(rev), old_merge)
-
-    if not upstream_lost and not upstream_gain:
-      # Trivially no changes caused by the upstream.
-      #
+    if not upstream_gain and cnt_mine == len(local_changes):
       return
 
     if self.IsDirty(consider_untracked=False):
       syncbuf.fail(self, _DirtyError())
       return
 
-    if upstream_lost:
+    # If the upstream switched on us, warn the user.
+    #
+    if branch.merge != self.revisionExpr:
+      if branch.merge and self.revisionExpr:
+        syncbuf.info(self,
+                     'manifest switched %s...%s',
+                     branch.merge,
+                     self.revisionExpr)
+      elif branch.merge:
+        syncbuf.info(self,
+                     'manifest no longer tracks %s',
+                     branch.merge)
+
+    if cnt_mine < len(local_changes):
       # Upstream rebased.  Not everything in HEAD
-      # may have been caused by the user.
+      # was created by this user.
       #
       syncbuf.info(self,
                    "discarding %d commits removed from upstream",
-                   len(upstream_lost))
+                   len(local_changes) - cnt_mine)
 
-    branch.remote = rem
-    branch.merge = self.revision
+    branch.remote = self.GetRemote(self.remote.name)
+    branch.merge = self.revisionExpr
     branch.Save()
 
-    my_changes = self._revlist(not_rev(old_merge), HEAD)
-    if my_changes:
+    if cnt_mine > 0:
       def _dorebase():
-        self._Rebase(upstream = old_merge, onto = rev)
+        self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
         self._CopyFiles()
       syncbuf.later2(self, _dorebase)
-    elif upstream_lost:
+    elif local_changes:
       try:
-        self._ResetHard(rev)
+        self._ResetHard(revid)
         self._CopyFiles()
       except GitError, e:
         syncbuf.fail(self, e)
         return
     else:
       def _doff():
-        self._FastForward(rev)
+        self._FastForward(revid)
         self._CopyFiles()
       syncbuf.later1(self, _doff)
 
@@ -731,7 +801,7 @@ class Project(object):
     if GitCommand(self, cmd, bare=True).Wait() != 0:
       return None
     return DownloadedChange(self,
-                            remote.ToLocal(self.revision),
+                            self.GetRevisionId(),
                             change_id,
                             patch_id,
                             self.bare_git.rev_parse('FETCH_HEAD'))
@@ -742,61 +812,113 @@ class Project(object):
   def StartBranch(self, name):
     """Create a new branch off the manifest's revision.
     """
-    try:
-      self.bare_git.rev_parse(R_HEADS + name)
-      exists = True
-    except GitError:
-      exists = False;
+    head = self.work_git.GetHead()
+    if head == (R_HEADS + name):
+      return True
 
-    if exists:
-      if name == self.CurrentBranch:
-        return True
-      else:
-        cmd = ['checkout', name, '--']
-        return GitCommand(self, cmd).Wait() == 0
+    all = self.bare_ref.all
+    if (R_HEADS + name) in all:
+      return GitCommand(self,
+                        ['checkout', name, '--'],
+                        capture_stdout = True,
+                        capture_stderr = True).Wait() == 0
 
-    else:
-      branch = self.GetBranch(name)
-      branch.remote = self.GetRemote(self.remote.name)
-      branch.merge = self.revision
+    branch = self.GetBranch(name)
+    branch.remote = self.GetRemote(self.remote.name)
+    branch.merge = self.revisionExpr
+    revid = self.GetRevisionId(all)
 
-      rev = branch.LocalMerge
-      cmd = ['checkout', '-b', branch.name, rev]
-      if GitCommand(self, cmd).Wait() == 0:
-        branch.Save()
-        return True
-      else:
-        return False
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid and head and revid == head:
+      ref = os.path.join(self.gitdir, R_HEADS + name)
+      try:
+        os.makedirs(os.path.dirname(ref))
+      except OSError:
+        pass
+      _lwrite(ref, '%s\n' % revid)
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      branch.Save()
+      return True
+
+    if GitCommand(self,
+                  ['checkout', '-b', branch.name, revid],
+                  capture_stdout = True,
+                  capture_stderr = True).Wait() == 0:
+      branch.Save()
+      return True
+    return False
 
   def CheckoutBranch(self, name):
     """Checkout a local topic branch.
     """
+    rev = R_HEADS + name
+    head = self.work_git.GetHead()
+    if head == rev:
+      # Already on the branch
+      #
+      return True
 
-    # Be sure the branch exists
+    all = self.bare_ref.all
     try:
-      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
-    except GitError:
-      return False;
+      revid = all[rev]
+    except KeyError:
+      # Branch does not exist in this project
+      #
+      return False
 
-    # Do the checkout
-    cmd = ['checkout', name, '--']
-    return GitCommand(self, cmd).Wait() == 0
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if head == revid:
+      # Same revision; just update HEAD to point to the new
+      # target branch, but otherwise take no other action.
+      #
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      return True
+
+    return GitCommand(self,
+                      ['checkout', name, '--'],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
     """
-    try:
-      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
-    except GitError:
-      return
+    rev = R_HEADS + name
+    all = self.bare_ref.all
+    if rev not in all:
+      # Doesn't exist; assume already abandoned.
+      #
+      return True
 
-    if self.CurrentBranch == name:
-      self._Checkout(
-        self.GetRemote(self.remote.name).ToLocal(self.revision),
-        quiet=True)
+    head = self.work_git.GetHead()
+    if head == rev:
+      # We can't destroy the branch while we are sitting
+      # on it.  Switch to a detached HEAD.
+      #
+      head = all[head]
 
-    cmd = ['branch', '-D', name]
-    GitCommand(self, cmd, capture_stdout=True).Wait()
+      revid = self.GetRevisionId(all)
+      if head == revid:
+        _lwrite(os.path.join(self.worktree, '.git', HEAD),
+                '%s\n' % revid)
+      else:
+        self._Checkout(revid, quiet=True)
+
+    return GitCommand(self,
+                      ['branch', '-D', name],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def PruneHeads(self):
     """Prune any topic branches already merged into upstream.
@@ -810,7 +932,7 @@ class Project(object):
         if cb is None or name != cb:
           kill.append(name)
 
-    rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
+    rev = self.GetRevisionId(left)
     if cb is not None \
        and not self._revlist(HEAD + '...' + rev) \
        and not self.IsDirty(consider_untracked = False):
@@ -818,9 +940,8 @@ class Project(object):
       kill.append(cb)
 
     if kill:
-      try:
-        old = self.bare_git.GetHead()
-      except GitError:
+      old = self.bare_git.GetHead()
+      if old is None:
         old = 'refs/heads/please_never_use_this_as_a_branch_name'
 
       try:
@@ -861,11 +982,19 @@ class Project(object):
   def _RemoteFetch(self, name=None):
     if not name:
       name = self.remote.name
+
+    ssh_proxy = False
+    if self.GetRemote(name).PreConnectFetch():
+      ssh_proxy = True
+
     cmd = ['fetch']
     if not self.worktree:
       cmd.append('--update-head-ok')
     cmd.append(name)
-    return GitCommand(self, cmd, bare = True).Wait() == 0
+    return GitCommand(self,
+                      cmd,
+                      bare = True,
+                      ssh_proxy = ssh_proxy).Wait() == 0
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -939,17 +1068,11 @@ class Project(object):
           raise
 
   def _InitRemote(self):
-    if self.remote.fetchUrl:
+    if self.remote.url:
       remote = self.GetRemote(self.remote.name)
-
-      url = self.remote.fetchUrl
-      while url.endswith('/'):
-        url = url[:-1]
-      url += '/%s.git' % self.name
-      remote.url = url
-      remote.review = self.remote.reviewUrl
-      if remote.projectname is None:
-        remote.projectname = self.name
+      remote.url = self.remote.url
+      remote.review = self.remote.review
+      remote.projectname = self.name
 
       if self.worktree:
         remote.ResetFetch(mirror=False)
@@ -957,34 +1080,27 @@ class Project(object):
         remote.ResetFetch(mirror=True)
       remote.Save()
 
-    for r in self.extraRemotes.values():
-      remote = self.GetRemote(r.name)
-      remote.url = r.fetchUrl
-      remote.review = r.reviewUrl
-      if r.projectName:
-        remote.projectname = r.projectName
-      elif remote.projectname is None:
-        remote.projectname = self.name
-      remote.ResetFetch()
-      remote.Save()
-
   def _InitMRef(self):
     if self.manifest.branch:
-      msg = 'manifest set to %s' % self.revision
-      ref = R_M + self.manifest.branch
-
-      if IsId(self.revision):
-        dst = self.revision + '^0'
-        self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
-      else:
-        remote = self.GetRemote(self.remote.name)
-        dst = remote.ToLocal(self.revision)
-        self.bare_git.symbolic_ref('-m', msg, ref, dst)
+      self._InitAnyMRef(R_M + self.manifest.branch)
 
   def _InitMirrorHead(self):
-    dst = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    msg = 'manifest set to %s' % self.revision
-    self.bare_git.SetHead(dst, message=msg)
+    self._InitAnyMRef(HEAD)
+
+  def _InitAnyMRef(self, ref):
+    cur = self.bare_ref.symref(ref)
+
+    if self.revisionId:
+      if cur != '' or self.bare_ref.get(ref) != self.revisionId:
+        msg = 'manifest set to %s' % self.revisionId
+        dst = self.revisionId + '^0'
+        self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
+    else:
+      remote = self.GetRemote(self.remote.name)
+      dst = remote.ToLocal(self.revisionExpr)
+      if cur != dst:
+        msg = 'manifest set to %s' % self.revisionExpr
+        self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
   def _InitWorkTree(self):
     dotgit = os.path.join(self.worktree, '.git')
@@ -1011,56 +1127,33 @@ class Project(object):
           else:
             raise
 
-      rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-      rev = self.bare_git.rev_parse('%s^0' % rev)
-
-      f = open(os.path.join(dotgit, HEAD), 'wb')
-      f.write("%s\n" % rev)
-      f.close()
+      _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
       cmd = ['read-tree', '--reset', '-u']
       cmd.append('-v')
-      cmd.append('HEAD')
+      cmd.append(HEAD)
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
+      self._CopyFiles()
 
   def _gitdir_path(self, path):
     return os.path.join(self.gitdir, path)
 
-  def _revlist(self, *args):
-    cmd = []
-    cmd.extend(args)
-    cmd.append('--')
-    return self.work_git.rev_list(*args)
+  def _revlist(self, *args, **kw):
+    a = []
+    a.extend(args)
+    a.append('--')
+    return self.work_git.rev_list(*a, **kw)
 
   @property
   def _allrefs(self):
-    return self.bare_git.ListRefs()
+    return self.bare_ref.all
 
   class _GitGetByExec(object):
     def __init__(self, project, bare):
       self._project = project
       self._bare = bare
 
-    def ListRefs(self, *args):
-      cmdv = ['for-each-ref', '--format=%(objectname) %(refname)']
-      cmdv.extend(args)
-      p = GitCommand(self._project,
-                     cmdv,
-                     bare = self._bare,
-                     capture_stdout = True,
-                     capture_stderr = True)
-      r = {}
-      for line in p.process.stdout:
-        id, name = line[:-1].split(' ', 2)
-        r[name] = id
-      if p.Wait() != 0:
-        raise GitError('%s for-each-ref %s: %s' % (
-                       self._project.name,
-                       str(args),
-                       p.stderr))
-      return r
-
     def LsOthers(self):
       p = GitCommand(self._project,
                      ['ls-files',
@@ -1126,7 +1219,18 @@ class Project(object):
         p.Wait()
 
     def GetHead(self):
-      return self.symbolic_ref(HEAD)
+      if self._bare:
+        path = os.path.join(self._project.gitdir, HEAD)
+      else:
+        path = os.path.join(self._project.worktree, '.git', HEAD)
+      fd = open(path, 'rb')
+      try:
+        line = fd.read()
+      finally:
+        fd.close()
+      if line.startswith('ref: '):
+        return line[5:-1]
+      return line[:-1]
 
     def SetHead(self, ref, message=None):
       cmdv = []
@@ -1162,9 +1266,13 @@ class Project(object):
       if not old:
         old = self.rev_parse(name)
       self.update_ref('-d', name, old)
+      self._project.bare_ref.deleted(name)
 
-    def rev_list(self, *args):
-      cmdv = ['rev-list']
+    def rev_list(self, *args, **kw):
+      if 'format' in kw:
+        cmdv = ['log', '--pretty=format:%s' % kw['format']]
+      else:
+        cmdv = ['rev-list']
       cmdv.extend(args)
       p = GitCommand(self._project,
                      cmdv,
@@ -1173,7 +1281,9 @@ class Project(object):
                      capture_stderr = True)
       r = []
       for line in p.process.stdout:
-        r.append(line[:-1])
+        if line[-1] == '\n':
+          line = line[:-1]
+        r.append(line)
       if p.Wait() != 0:
         raise GitError('%s rev-list %s: %s' % (
                        self._project.name,
@@ -1320,9 +1430,10 @@ class MetaProject(Project):
                      name = name,
                      gitdir = gitdir,
                      worktree = worktree,
-                     remote = Remote('origin'),
+                     remote = RemoteSpec('origin'),
                      relpath = '.repo/%s' % name,
-                     revision = 'refs/heads/master')
+                     revisionExpr = 'refs/heads/master',
+                     revisionId = None)
 
   def PreSync(self):
     if self.Exists:
@@ -1330,13 +1441,35 @@ class MetaProject(Project):
       if cb:
         base = self.GetBranch(cb).merge
         if base:
-          self.revision = base
+          self.revisionExpr = base
+          self.revisionId = None
+
+  @property
+  def LastFetch(self):
+    try:
+      fh = os.path.join(self.gitdir, 'FETCH_HEAD')
+      return os.path.getmtime(fh)
+    except OSError:
+      return 0
 
   @property
   def HasChanges(self):
     """Has the remote received new commits not yet checked out?
     """
-    rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    if self._revlist(not_rev(HEAD), rev):
+    if not self.remote or not self.revisionExpr:
+      return False
+
+    all = self.bare_ref.all
+    revid = self.GetRevisionId(all)
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid == head:
+      return False
+    elif self._revlist(not_rev(HEAD), revid):
       return True
     return False

repo

@@ -120,7 +120,7 @@ group.add_option('--mirror',
                  help='mirror the forrest')
 
 # Tool
-group = init_optparse.add_option_group('Version options')
+group = init_optparse.add_option_group('repo Version options')
 group.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')
@@ -505,7 +505,7 @@ def _RunSelf(wrapper_path):
   my_git = os.path.join(my_dir, '.git')
 
   if os.path.isfile(my_main) and os.path.isdir(my_git):
-    for name in ['manifest.py',
+    for name in ['git_config.py',
                  'project.py',
                  'subcmds']:
       if not os.path.exists(os.path.join(my_dir, name)):

subcmds/abandon.py

@@ -16,6 +16,7 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Abandon(Command):
   common = True
@@ -38,5 +39,23 @@ It is equivalent to "git branch -D <branchname>".
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
-    for project in self.GetProjects(args[1:]):
-      project.AbandonBranch(nb)
+    nb = args[0]
+    err = []
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Abandon %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.AbandonBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot abandon %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/branches.py

@@ -61,12 +61,34 @@ class Branches(Command):
 %prog [<project>...]
 
 Summarizes the currently available topic branches.
-"""
 
-  def _Options(self, p):
-    p.add_option('-a', '--all',
-                 dest='all', action='store_true',
-                 help='show all branches, not just the majority')
+Branch Display
+--------------
+
+The branch display output by this command is organized into four
+columns of information; for example:
+
+ *P nocolor                   | in repo
+    repo2                     |
+
+The first column contains a * if the branch is the currently
+checked out branch in any of the specified projects, or a blank
+if no project has the branch checked out.
+
+The second column contains either blank, p or P, depending upon
+the upload status of the branch.
+
+ (blank): branch not yet published by repo upload
+       P: all commits were published by repo upload
+       p: only some commits were published by repo upload
+
+The third column contains the branch name.
+
+The fourth column (after the | separator) lists the projects that
+the branch appears in, or does not appear in.  If no project list
+is shown, then the branch appears in all projects.
+
+"""
 
   def Execute(self, opt, args):
     projects = self.GetProjects(args)
@@ -84,17 +106,9 @@ Summarizes the currently available topic branches.
     names = all.keys()
     names.sort()
 
-    if not opt.all and not args:
-      # No -a and no specific projects listed; try to filter the
-      # results down to only the majority of projects.
-      #
-      n = []
-      for name in names:
-        i = all[name]
-        if i.IsCurrent \
-        or 80 <= (100 * len(i.projects)) / project_cnt:
-          n.append(name)
-      names = n
+    if not names:
+      print >>sys.stderr, '   (no branches)'
+      return
 
     width = 25
     for name in names:
@@ -122,7 +136,7 @@ Summarizes the currently available topic branches.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
-      if in_cnt < project_cnt and (in_cnt == 1 or opt.all):
+      if in_cnt < project_cnt and (in_cnt == 1):
         fmt = out.write
         paths = []
         if in_cnt < project_cnt - in_cnt: 

subcmds/checkout.py

@@ -15,6 +15,7 @@
 
 import sys
 from command import Command
+from progress import Progress
 
 class Checkout(Command):
   common = True
@@ -35,13 +36,23 @@ The command is equivalent to:
     if not args:
       self.Usage()
 
-    retValue = 0;
+    nb = args[0]
+    err = []
+    all = self.GetProjects(args[1:])
 
-    branch = args[0]
-    for project in self.GetProjects(args[1:]):
-      if not project.CheckoutBranch(branch):
-        retValue = 1;
-        print >>sys.stderr, "error: checking out branch '%s' in %s failed" % (branch, project.name)
+    pm = Progress('Checkout %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.CheckoutBranch(nb):
+        err.append(project)
+    pm.end()
 
-    if (retValue != 0):
-      sys.exit(retValue);
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot checkout %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/forall.py

@@ -13,12 +13,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import fcntl
 import re
 import os
+import select
 import sys
 import subprocess
+
+from color import Coloring
 from command import Command, MirrorSafeCommand
 
+_CAN_COLOR = [
+  'branch',
+  'diff',
+  'grep',
+  'log',
+]
+
+class ForallColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'forall')
+    self.project = self.printer('project', attr='bold')
+
+
 class Forall(Command, MirrorSafeCommand):
   common = False
   helpSummary = "Run a shell command in each project"
@@ -28,6 +45,24 @@ class Forall(Command, MirrorSafeCommand):
   helpDescription = """
 Executes the same shell command in each project.
 
+Output Formatting
+-----------------
+
+The -p option causes '%prog' to bind pipes to the command's stdin,
+stdout and stderr streams, and pipe all output into a continuous
+stream that is displayed in a single pager session.  Project headings
+are inserted before the output of each command is displayed.  If the
+command produces no output in a project, no heading is displayed.
+
+The formatting convention used by -p is very suitable for some
+types of searching, e.g. `repo forall -p -c git log -SFoo` will
+print all commits that add or remove references to Foo.
+
+The -v option causes '%prog' to display stderr messages if a
+command produces output only on stderr.  Normally the -p option
+causes command output to be suppressed until the command produces
+at least one byte of output on stdout.
+
 Environment
 -----------
 
@@ -50,8 +85,8 @@ as written in the manifest.
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
-stdin, stdout, stderr are inherited from the terminal and are
-not redirected.
+Unless -p is used, stdin, stdout, stderr are inherited from the
+terminal and are not redirected.
 """
 
   def _Options(self, p):
@@ -65,6 +100,17 @@ not redirected.
                  action='callback',
                  callback=cmd)
 
+    g = p.add_option_group('Output')
+    g.add_option('-p',
+                 dest='project_header', action='store_true',
+                 help='Show project headers before output')
+    g.add_option('-v', '--verbose',
+                 dest='verbose', action='store_true',
+                 help='Show command error messages')
+
+  def WantPager(self, opt):
+    return opt.project_header
+
   def Execute(self, opt, args):
     if not opt.command:
       self.Usage()
@@ -79,8 +125,31 @@ not redirected.
       cmd.append(cmd[0])
     cmd.extend(opt.command[1:])
 
+    if  opt.project_header \
+    and not shell \
+    and cmd[0] == 'git':
+      # If this is a direct git command that can enable colorized
+      # output and the user prefers coloring, add --color into the
+      # command line because we are going to wrap the command into
+      # a pipe and git won't know coloring should activate.
+      #
+      for cn in cmd[1:]:
+        if not cn.startswith('-'):
+          break
+      if cn in _CAN_COLOR:
+        class ColorCmd(Coloring):
+          def __init__(self, config, cmd):
+            Coloring.__init__(self, config, cmd)
+        if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
+          cmd.insert(cmd.index(cn) + 1, '--color')
+
     mirror = self.manifest.IsMirror
+    out = ForallColoring(self.manifest.manifestProject.config)
+    out.redirect(sys.stdout)
+
     rc = 0
+    first = True
+
     for project in self.GetProjects(args):
       env = dict(os.environ.iteritems())
       def setenv(name, val):
@@ -91,10 +160,8 @@ not redirected.
       setenv('REPO_PROJECT', project.name)
       setenv('REPO_PATH', project.relpath)
       setenv('REPO_REMOTE', project.remote.name)
-      setenv('REPO_LREV', project\
-        .GetRemote(project.remote.name)\
-        .ToLocal(project.revision))
-      setenv('REPO_RREV', project.revision)
+      setenv('REPO_LREV', project.GetRevisionId())
+      setenv('REPO_RREV', project.revisionExpr)
 
       if mirror:
         setenv('GIT_DIR', project.gitdir)
@@ -102,10 +169,76 @@ not redirected.
       else:
         cwd = project.worktree
 
+      if opt.project_header:
+        stdin = subprocess.PIPE
+        stdout = subprocess.PIPE
+        stderr = subprocess.PIPE
+      else:
+        stdin = None
+        stdout = None
+        stderr = None
+
       p = subprocess.Popen(cmd,
                            cwd = cwd,
                            shell = shell,
-                           env = env)
+                           env = env,
+                           stdin = stdin,
+                           stdout = stdout,
+                           stderr = stderr)
+
+      if opt.project_header:
+        class sfd(object):
+          def __init__(self, fd, dest):
+            self.fd = fd
+            self.dest = dest
+          def fileno(self):
+            return self.fd.fileno()
+
+        empty = True
+        didout = False
+        errbuf = ''
+
+        p.stdin.close()
+        s_in = [sfd(p.stdout, sys.stdout),
+                sfd(p.stderr, sys.stderr)]
+
+        for s in s_in:
+          flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+          fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+        while s_in:
+          in_ready, out_ready, err_ready = select.select(s_in, [], [])
+          for s in in_ready:
+            buf = s.fd.read(4096)
+            if not buf:
+              s.fd.close()
+              s_in.remove(s)
+              continue
+
+            if not opt.verbose:
+              if s.fd == p.stdout:
+                didout = True
+              else:
+                errbuf += buf
+                continue
+
+            if empty:
+              if first:
+                first = False
+              else:
+                out.nl()
+              out.project('project %s/', project.relpath)
+              out.nl()
+              out.flush()
+              if errbuf:
+                sys.stderr.write(errbuf)
+                sys.stderr.flush()
+                errbuf = ''
+              empty = False
+
+            s.dest.write(buf)
+            s.dest.flush()
+
       r = p.wait()
       if r != 0 and r != rc:
         rc = r

subcmds/grep.py

@@ -17,7 +17,7 @@ import sys
 from optparse import SUPPRESS_HELP
 from color import Coloring
 from command import PagedCommand
-from git_command import GitCommand
+from git_command import git_require, GitCommand
 
 class GrepColoring(Coloring):
   def __init__(self, config):
@@ -33,8 +33,8 @@ class Grep(PagedCommand):
   helpDescription = """
 Search for the specified patterns in all project files.
 
-Options
--------
+Boolean Options
+---------------
 
 The following options can appear as often as necessary to express
 the pattern to locate:
@@ -158,7 +158,7 @@ contain a line that matches both expressions:
     out = GrepColoring(self.manifest.manifestProject.config)
 
     cmd_argv = ['grep']
-    if out.is_on:
+    if out.is_on and git_require((1,6,3)):
       cmd_argv.append('--color')
     cmd_argv.extend(getattr(opt,'cmd_argv',[]))
 

subcmds/help.py

@@ -107,7 +107,7 @@ See 'repo help --all' for a complete list of recognized commands.
         body = body.strip()
         body = body.replace('%prog', me)
 
-        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n(={2,}|-{2,})$')
+        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n([=~-]{2,})$')
         for para in body.split("\n\n"):
           if para.startswith(' '):
             self.write('%s', para)
@@ -117,9 +117,19 @@ See 'repo help --all' for a complete list of recognized commands.
 
           m = asciidoc_hdr.match(para)
           if m:
-            self.heading('%s', m.group(1))
+            title = m.group(1)
+            type = m.group(2)
+            if type[0] in ('=', '-'):
+              p = self.heading
+            else:
+              def _p(fmt, *args):
+                self.write('  ')
+                self.heading(fmt, *args)
+              p = _p
+
+            p('%s', title)
             self.nl()
-            self.heading('%s', ''.ljust(len(m.group(1)),'-'))
+            p('%s', ''.ljust(len(title),type[0]))
             self.nl()
             continue
 
@@ -128,8 +138,8 @@ See 'repo help --all' for a complete list of recognized commands.
         self.wrap.end_paragraph(0)
 
     out = _Out(self.manifest.globalConfig)
-    cmd.OptionParser.print_help()
     out._PrintSection('Summary', 'helpSummary')
+    cmd.OptionParser.print_help()
     out._PrintSection('Description', 'helpDescription')
 
   def _Options(self, p):

subcmds/init.py

@@ -19,9 +19,8 @@ import sys
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
-from remote import Remote
 from project import SyncBuffer
-from git_command import git, MIN_GIT_VERSION
+from git_command import git_require, MIN_GIT_VERSION
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
@@ -35,9 +34,20 @@ The latest repo source code and manifest collection is downloaded
 from the server and is installed in the .repo/ directory in the
 current working directory.
 
-The optional <manifest> argument can be used to specify an alternate
-manifest to be used. If no manifest is specified, the manifest
-default.xml will be used.
+The optional -b argument can be used to select the manifest branch
+to checkout and use.  If no branch is specified, master is assumed.
+
+The optional -m argument can be used to specify an alternate manifest
+to be used. If no manifest is specified, the manifest default.xml
+will be used.
+
+Switching Manifest Branches
+---------------------------
+
+To switch to another manifest branch, `repo init -b otherbranch`
+may be used in an existing client.  However, as this only updates the
+manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
+to update the working directory files.
 """
 
   def _Options(self, p):
@@ -64,7 +74,7 @@ default.xml will be used.
 
 
     # Tool
-    g = p.add_option_group('Version options')
+    g = p.add_option_group('repo Version options')
     g.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')
@@ -75,19 +85,6 @@ default.xml will be used.
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
-  def _CheckGitVersion(self):
-    ver_str = git.version()
-    if not ver_str.startswith('git version '):
-      print >>sys.stderr, 'error: "%s" unsupported' % ver_str
-      sys.exit(1)
-
-    ver_str = ver_str[len('git version '):].strip()
-    ver_act = tuple(map(lambda x: int(x), ver_str.split('.')[0:3]))
-    if ver_act < MIN_GIT_VERSION:
-      need = '.'.join(map(lambda x: str(x), MIN_GIT_VERSION))
-      print >>sys.stderr, 'fatal: git %s or later required' % need
-      sys.exit(1)
-
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
@@ -103,12 +100,12 @@ default.xml will be used.
       m._InitGitDir()
 
       if opt.manifest_branch:
-        m.revision = opt.manifest_branch
+        m.revisionExpr = opt.manifest_branch
       else:
-        m.revision = 'refs/heads/master'
+        m.revisionExpr = 'refs/heads/master'
     else:
       if opt.manifest_branch:
-        m.revision = opt.manifest_branch
+        m.revisionExpr = opt.manifest_branch
       else:
         m.PreSync()
 
@@ -204,7 +201,7 @@ default.xml will be used.
       gc.SetString('color.ui', 'auto')
 
   def Execute(self, opt, args):
-    self._CheckGitVersion()
+    git_require(MIN_GIT_VERSION, fail=True)
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 

subcmds/selfupdate.py

@@ -35,10 +35,11 @@ need to be performed by an end-user.
 """
 
   def _Options(self, p):
-    p.add_option('--no-repo-verify',
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
-    p.add_option('--repo-upgraded',
+    g.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 

subcmds/stage.py

@@ -55,12 +55,12 @@ The '%prog' command stages files to prepare the next commit.
 
     out = _ProjectList(self.manifest.manifestProject.config)
     while True:
-      out.header('        %-20s %s', 'project', 'path')
+      out.header('        %s', 'project')
       out.nl()
 
       for i in xrange(0, len(all)):
         p = all[i]
-        out.write('%3d:    %-20s %s', i + 1, p.name, p.relpath + '/')
+        out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
       out.nl()
 

subcmds/start.py

@@ -16,27 +16,23 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Start(Command):
   common = True
   helpSummary = "Start a new branch for development"
   helpUsage = """
-%prog <newbranchname> [<project>...]
-
-This subcommand starts a new branch of development that is automatically
-pulled from a remote branch.
-
-It is equivalent to the following git commands:
-
-"git branch --track <newbranchname> m/<codeline>",
-or 
-"git checkout --track -b <newbranchname> m/<codeline>".
-
-All three forms set up the config entries that repo bases some of its
-processing on.  Use %prog or git branch or checkout with --track to ensure
-the configuration data is set up properly.
-
+%prog <newbranchname> [--all | <project>...]
 """
+  helpDescription = """
+'%prog' begins a new branch of development, starting from the
+revision specified in the manifest.
+"""
+
+  def _Options(self, p):
+    p.add_option('--all',
+                 dest='all', action='store_true',
+                 help='begin branch in all projects')
 
   def Execute(self, opt, args):
     if not args:
@@ -48,12 +44,25 @@ the configuration data is set up properly.
       sys.exit(1)
 
     err = []
-    for project in self.GetProjects(args[1:]):
+    projects = []
+    if not opt.all:
+      projects = args[1:]
+      if len(projects) < 1:
+        print >>sys.stderr, "error: at least one project must be specified"
+        sys.exit(1)
+
+    all = self.GetProjects(projects)
+
+    pm = Progress('Starting %s' % nb, len(all))
+    for project in all:
+      pm.update()
       if not project.StartBranch(nb):
         err.append(project)
+    pm.end()
 
     if err:
-      err.sort()
       for p in err:
-        print >>sys.stderr, "error: cannot start in %s" % p.relpath
+        print >>sys.stderr,\
+          "error: %s/: cannot start %s" \
+          % (p.relpath, nb)
       sys.exit(1)

subcmds/status.py

@@ -64,6 +64,19 @@ the following meanings:
     all = self.GetProjects(args)
     clean = 0
 
+    on = {}
+    for project in all:
+      cb = project.CurrentBranch
+      if cb:
+        if cb not in on:
+          on[cb] = []
+        on[cb].append(project)
+
+    branch_names = list(on.keys())
+    branch_names.sort()
+    for cb in branch_names:
+      print '# on branch %s' % cb
+
     for project in all:
       state = project.PrintWorkTreeStatus()
       if state == 'CLEAN':

subcmds/sync.py

@@ -16,11 +16,15 @@
 from optparse import SUPPRESS_HELP
 import os
 import re
+import shutil
 import subprocess
 import sys
+import time
 
 from git_command import GIT
 from project import HEAD
+from project import Project
+from project import RemoteSpec
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
 from project import R_HEADS
@@ -52,6 +56,35 @@ The -d/--detach option can be used to switch specified projects
 back to the manifest revision.  This option is especially helpful
 if the project is currently on a topic branch, but the manifest
 revision is temporarily needed.
+
+SSH Connections
+---------------
+
+If at least one project remote URL uses an SSH connection (ssh://,
+git+ssh://, or user@host:path syntax) repo will automatically
+enable the SSH ControlMaster option when connecting to that host.
+This feature permits other projects in the same '%prog' session to
+reuse the same SSH tunnel, saving connection setup overheads.
+
+To disable this behavior on UNIX platforms, set the GIT_SSH
+environment variable to 'ssh'.  For example:
+
+  export GIT_SSH=ssh
+  %prog
+
+Compatibility
+~~~~~~~~~~~~~
+
+This feature is automatically disabled on Windows, due to the lack
+of UNIX domain socket support.
+
+This feature is not compatible with url.insteadof rewrites in the
+user's ~/.gitconfig.  '%prog' is currently not able to perform the
+rewrite early enough to establish the ControlMaster tunnel.
+
+If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
+later is required to fix a server side protocol bug.
+
 """
 
   def _Options(self, p):
@@ -65,14 +98,15 @@ revision is temporarily needed.
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
 
-    p.add_option('--no-repo-verify',
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
-    p.add_option('--repo-upgraded',
+    g.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _Fetch(self, *projects):
+  def _Fetch(self, projects):
     fetched = set()
     pm = Progress('Fetching projects', len(projects))
     for project in projects:
@@ -86,6 +120,61 @@ revision is temporarily needed.
     pm.end()
     return fetched
 
+  def UpdateProjectList(self):
+    new_project_paths = []
+    for project in self.manifest.projects.values():
+      if project.relpath:
+        new_project_paths.append(project.relpath)
+    file_name = 'project.list'
+    file_path = os.path.join(self.manifest.repodir, file_name)
+    old_project_paths = []
+
+    if os.path.exists(file_path):
+      fd = open(file_path, 'r')
+      try:
+        old_project_paths = fd.read().split('\n')
+      finally:
+        fd.close()
+      for path in old_project_paths:
+        if not path:
+          continue
+        if path not in new_project_paths:
+          project = Project(
+                         manifest = self.manifest,
+                         name = path,
+                         remote = RemoteSpec('origin'),
+                         gitdir = os.path.join(self.manifest.topdir,
+                                               path, '.git'),
+                         worktree = os.path.join(self.manifest.topdir, path),
+                         relpath = path,
+                         revisionExpr = 'HEAD',
+                         revisionId = None)
+          if project.IsDirty():
+            print >>sys.stderr, 'error: Cannot remove project "%s": \
+uncommitted changes are present' % project.relpath
+            print >>sys.stderr, '       commit changes, then run sync again'
+            return -1
+          else:
+            print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
+            shutil.rmtree(project.worktree)
+            # Try deleting parent subdirs if they are empty
+            dir = os.path.dirname(project.worktree)
+            while dir != self.manifest.topdir:
+              try:
+                os.rmdir(dir)
+              except OSError:
+                break
+              dir = os.path.dirname(dir)
+
+    new_project_paths.sort()
+    fd = open(file_path, 'w')
+    try:
+      fd.write('\n'.join(new_project_paths))
+      fd.write('\n')
+    finally:
+      fd.close()
+    return 0
+
   def Execute(self, opt, args):
     if opt.network_only and opt.detach_head:
       print >>sys.stderr, 'error: cannot combine -n and -d'
@@ -106,7 +195,14 @@ revision is temporarily needed.
     all = self.GetProjects(args, missing_ok=True)
 
     if not opt.local_only:
-      fetched = self._Fetch(rp, mp, *all)
+      to_fetch = []
+      now = time.time()
+      if (24 * 60 * 60) <= (now - rp.LastFetch):
+        to_fetch.append(rp)
+      to_fetch.append(mp)
+      to_fetch.extend(all)
+
+      fetched = self._Fetch(to_fetch)
       _PostRepoFetch(rp, opt.no_repo_verify)
       if opt.network_only:
         # bail out now; the rest touches the working tree
@@ -124,7 +220,14 @@ revision is temporarily needed.
         for project in all:
           if project.gitdir not in fetched:
             missing.append(project)
-        self._Fetch(*missing)
+        self._Fetch(missing)
+
+    if self.manifest.IsMirror:
+      # bail out now, we have no working tree
+      return
+
+    if self.UpdateProjectList():
+      sys.exit(1)
 
     syncbuf = SyncBuffer(mp.config,
                          detach_head = opt.detach_head)
@@ -169,17 +272,14 @@ def _VerifyTag(project):
 warning: Cannot automatically authenticate repo."""
     return True
 
-  remote = project.GetRemote(project.remote.name)
-  ref = remote.ToLocal(project.revision)
-
   try:
-    cur = project.bare_git.describe(ref)
+    cur = project.bare_git.describe(project.GetRevisionId())
   except GitError:
     cur = None
 
   if not cur \
      or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
-    rev = project.revision
+    rev = project.revisionExpr
     if rev.startswith(R_HEADS):
       rev = rev[len(R_HEADS):]
 

subcmds/upload.py

@@ -38,22 +38,21 @@ class Upload(InteractiveCommand):
 %prog [--re --cc] {[<project>]... | --replace <project>}
 """
   helpDescription = """
-The '%prog' command is used to send changes to the Gerrit code
-review system.  It searches for changes in local projects that do
-not yet exist in the corresponding remote repository.  If multiple
-changes are found, '%prog' opens an editor to allow the
-user to choose which change to upload.  After a successful upload,
-repo prints the URL for the change in the Gerrit code review system.
+The '%prog' command is used to send changes to the Gerrit Code
+Review system.  It searches for topic branches in local projects
+that have not yet been published for review.  If multiple topic
+branches are found, '%prog' opens an editor to allow the user to
+select which branches to upload.
 
-'%prog' searches for uploadable changes in all projects listed
-at the command line.  Projects can be specified either by name, or
-by a relative or absolute path to the project's local directory. If
-no projects are specified, '%prog' will search for uploadable
-changes in all projects listed in the manifest.
+'%prog' searches for uploadable changes in all projects listed at
+the command line.  Projects can be specified either by name, or by
+a relative or absolute path to the project's local directory. If no
+projects are specified, '%prog' will search for uploadable changes
+in all projects listed in the manifest.
 
 If the --reviewers or --cc options are passed, those emails are
 added to the respective list of users, and emails are sent to any
-new users.  Users passed to --reviewers must be already registered
+new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
 If the --replace option is passed the user can designate which
@@ -61,6 +60,33 @@ existing change(s) in Gerrit match up to the commits in the branch
 being uploaded.  For each matched pair of change,commit the commit
 will be added as a new patch set, completely replacing the set of
 files and description associated with the change in Gerrit.
+
+Configuration
+-------------
+
+review.URL.autoupload:
+
+To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+or global Git configuration option.  If review.URL.autoupload is set
+to "true" then repo will assume you always answer "y" at the prompt,
+and will not prompt you further.  If it is set to "false" then repo
+will assume you always answer "n", and will abort.
+
+The URL must match the review URL listed in the manifest XML file,
+or in the .git/config within the project.  For example:
+
+  [remote "origin"]
+    url = git://git.example.com/project.git
+    review = http://review.example.com/
+
+  [review "http://review.example.com/"]
+    autoupload = true
+
+References
+----------
+
+Gerrit Code Review:  http://code.google.com/p/gerrit/
+
 """
 
   def _Options(self, p):
@@ -77,21 +103,32 @@ files and description associated with the change in Gerrit.
   def _SingleBranch(self, branch, people):
     project = branch.project
     name = branch.name
-    date = branch.date
-    list = branch.commits
+    remote = project.GetBranch(name).remote
 
-    print 'Upload project %s/:' % project.relpath
-    print '  branch %s (%2d commit%s, %s):' % (
-                  name,
-                  len(list),
-                  len(list) != 1 and 's' or '',
-                  date)
-    for commit in list:
-      print '         %s' % commit
+    key = 'review.%s.autoupload' % remote.review
+    answer = project.config.GetBoolean(key)
 
-    sys.stdout.write('(y/n)? ')
-    answer = sys.stdin.readline().strip()
-    if answer in ('y', 'Y', 'yes', '1', 'true', 't'):
+    if answer is False:
+      _die("upload blocked by %s = false" % key)
+
+    if answer is None:
+      date = branch.date
+      list = branch.commits
+
+      print 'Upload project %s/:' % project.relpath
+      print '  branch %s (%2d commit%s, %s):' % (
+                    name,
+                    len(list),
+                    len(list) != 1 and 's' or '',
+                    date)
+      for commit in list:
+        print '         %s' % commit
+
+      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      answer = sys.stdin.readline().strip()
+      answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
+
+    if answer:
       self._UploadAndReport([branch], people)
     else:
       _die("upload aborted by user")
@@ -157,6 +194,18 @@ files and description associated with the change in Gerrit.
       _die("nothing uncommented for upload")
     self._UploadAndReport(todo, people)
 
+  def _FindGerritChange(self, branch):
+    last_pub = branch.project.WasPublished(branch.name)
+    if last_pub is None:
+      return ""
+
+    refs = branch.GetPublishedRefs()
+    try:
+      # refs/changes/XYZ/N --> XYZ
+      return refs.get(last_pub).split('/')[-2]
+    except:
+      return ""
+
   def _ReplaceBranch(self, project, people):
     branch = project.CurrentBranch
     if not branch:
@@ -169,8 +218,14 @@ files and description associated with the change in Gerrit.
 
     script = []
     script.append('# Replacing from branch %s' % branch.name)
-    for commit in branch.commits:
-      script.append('[      ] %s' % commit)
+
+    if len(branch.commits) == 1:
+      change = self._FindGerritChange(branch)
+      script.append('[%-6s] %s' % (change, branch.commits[0]))
+    else:
+      for commit in branch.commits:
+        script.append('[      ] %s' % commit)
+
     script.append('')
     script.append('# Insert change numbers in the brackets to add a new patch set.')
     script.append('# To create a new change record, leave the brackets empty.')

trace.py

@@ -13,13 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-class Remote(object):
-  def __init__(self, name,
-               fetch=None,
-               review=None,
-               projectName=None):
-    self.name = name
-    self.fetchUrl = fetch
-    self.reviewUrl = review
-    self.projectName = projectName
-    self.requiredCommits = []
+import sys
+import os
+REPO_TRACE = 'REPO_TRACE'
+
+try:
+  _TRACE = os.environ[REPO_TRACE] == '1'
+except KeyError:
+  _TRACE = False
+
+def IsTrace():
+  return _TRACE
+
+def SetTrace():
+  global _TRACE
+  _TRACE = True
+
+def Trace(fmt, *args):
+  if IsTrace():
+    print >>sys.stderr, fmt % args