Improve error handling when reading loose refs

When repo is trying to figure out branches the repository has by
traversing refs/heads, add exception handling for readline.

Change-Id: If3b2a3720c6496f52f629aa9a2539f186d6ec882

error.py

@@ -57,6 +57,15 @@ class UploadError(Exception):
   def __str__(self):
     return self.reason
 
+class DownloadError(Exception):
+  """Cannot download a repository.
+  """
+  def __init__(self, reason):
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
 class NoSuchProjectError(Exception):
   """A specified project does not exist in the work tree.
   """

git_command.py

@@ -221,26 +221,10 @@ class GitCommand(object):
     self.stdin = p.stdin
 
   def Wait(self):
-    p = self.process
-
-    if p.stdin:
-      p.stdin.close()
-      self.stdin = None
-
-    if p.stdout:
-      self.stdout = p.stdout.read()
-      p.stdout.close()
-    else:
-      p.stdout = None
-
-    if p.stderr:
-      self.stderr = p.stderr.read()
-      p.stderr.close()
-    else:
-      p.stderr = None
-
     try:
-      rc = p.wait()
+      p = self.process
+      (self.stdout, self.stderr) = p.communicate()
+      rc = p.returncode
     finally:
       _remove_ssh_client(p)
     return rc

git_config.py

@@ -26,7 +26,6 @@ import time
 import urllib2
 
 from signal import SIGTERM
-from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
 
@@ -491,6 +490,12 @@ def close_ssh():
 URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
 URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
 
+def GetSchemeFromUrl(url):
+  m = URI_ALL.match(url)
+  if m:
+    return m.group(1)
+  return None
+
 def _preconnect(url):
   m = URI_ALL.match(url)
   if m:
@@ -570,9 +575,19 @@ class Remote(object):
         self._review_protocol = info[0]
         self._review_host = info[1]
         self._review_port = info[2]
+      elif 'REPO_HOST_PORT_INFO' in os.environ:
+        info = os.environ['REPO_HOST_PORT_INFO']
+        self._review_protocol = 'ssh'
+        self._review_host = info.split(" ")[0]
+        self._review_port = info.split(" ")[1]
+
+        REVIEW_CACHE[u] = (
+          self._review_protocol,
+          self._review_host,
+          self._review_port)
       else:
         try:
-          info = urlopen(u).read()
+          info = urllib2.urlopen(u).read()
           if info == 'NOT_AVAILABLE':
             raise UploadError('%s: SSH disabled' % self.review)
           if '<' in info:
@@ -584,15 +599,15 @@ class Remote(object):
           self._review_protocol = 'ssh'
           self._review_host = info.split(" ")[0]
           self._review_port = info.split(" ")[1]
-        except urllib2.URLError, e:
-          raise UploadError('%s: %s' % (self.review, e.reason[1]))
-        except HTTPError, e:
+        except urllib2.HTTPError, e:
           if e.code == 404:
             self._review_protocol = 'http-post'
             self._review_host = None
             self._review_port = None
           else:
-            raise UploadError('Upload over ssh unavailable')
+            raise UploadError('Upload over SSH unavailable')
+        except urllib2.URLError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
 
         REVIEW_CACHE[u] = (
           self._review_protocol,

git_refs.py

@@ -139,13 +139,15 @@ class GitRefs(object):
   def _ReadLoose1(self, path, name):
     try:
       fd = open(path, 'rb')
-      mtime = os.path.getmtime(path)
-    except OSError:
-      return
-    except IOError:
+    except:
       return
+
     try:
-      id = fd.readline()
+      try:
+        mtime = os.path.getmtime(path)
+        id = fd.readline()
+      except:
+        return
     finally:
       fd.close()
 

main.py

@@ -37,6 +37,7 @@ from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
 from editor import Editor
+from error import DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoSuchProjectError
 from error import RepoChangedException
@@ -72,6 +73,7 @@ class _Repo(object):
     all_commands['branch'] = all_commands['branches']
 
   def _Run(self, argv):
+    result = 0
     name = None
     glob = []
 
@@ -95,7 +97,7 @@ class _Repo(object):
         name = 'version'
       else:
         print >>sys.stderr, 'fatal: invalid usage of --version'
-        sys.exit(1)
+        return 1
 
     try:
       cmd = self.commands[name]
@@ -103,7 +105,7 @@ class _Repo(object):
       print >>sys.stderr,\
             "repo: '%s' is not a repo command.  See 'repo help'."\
             % name
-      sys.exit(1)
+      return 1
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
@@ -113,7 +115,7 @@ class _Repo(object):
       print >>sys.stderr, \
             "fatal: '%s' requires a working directory"\
             % name
-      sys.exit(1)
+      return 1
 
     copts, cargs = cmd.OptionParser.parse_args(argv)
 
@@ -131,7 +133,7 @@ class _Repo(object):
     try:
       start = time.time()
       try:
-        cmd.Execute(copts, cargs)
+        result = cmd.Execute(copts, cargs)
       finally:
         elapsed = time.time() - start
         hours, remainder = divmod(elapsed, 3600)
@@ -143,15 +145,20 @@ class _Repo(object):
           else:
             print >>sys.stderr, 'real\t%dh%dm%.3fs' \
               % (hours, minutes, seconds)
+    except DownloadError, e:
+      print >>sys.stderr, 'error: %s' % str(e)
+      return 1
     except ManifestInvalidRevisionError, e:
       print >>sys.stderr, 'error: %s' % str(e)
-      sys.exit(1)
+      return 1
     except NoSuchProjectError, e:
       if e.name:
         print >>sys.stderr, 'error: project %s not found' % e.name
       else:
         print >>sys.stderr, 'error: no project in current directory'
-      sys.exit(1)
+      return 1
+
+    return result
 
 def _MyRepoPath():
   return os.path.dirname(__file__)
@@ -269,6 +276,24 @@ class _UserAgentHandler(urllib2.BaseHandler):
     req.add_header('User-Agent', _UserAgent())
     return req
 
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_auth_reqed(self, authreq, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+        self, authreq, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -283,7 +308,7 @@ def init_http():
     pass
   except IOError:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(_BasicAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
@@ -294,6 +319,8 @@ def init_http():
   urllib2.install_opener(urllib2.build_opener(*handlers))
 
 def _Main(argv):
+  result = 0
+
   opt = optparse.OptionParser(usage="repo wrapperinfo -- ...")
   opt.add_option("--repo-dir", dest="repodir",
                  help="path to .repo/")
@@ -312,11 +339,11 @@ def _Main(argv):
     try:
       init_ssh()
       init_http()
-      repo._Run(argv)
+      result = repo._Run(argv) or 0
     finally:
       close_ssh()
   except KeyboardInterrupt:
-    sys.exit(1)
+    result = 1
   except RepoChangedException, rce:
     # If repo changed, re-exec ourselves.
     #
@@ -327,7 +354,9 @@ def _Main(argv):
     except OSError, e:
       print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
       print >>sys.stderr, 'fatal: %s' % e
-      sys.exit(128)
+      result = 128
+
+  sys.exit(result)
 
 if __name__ == '__main__':
   _Main(sys.argv[1:])

manifest_xml.py

@@ -14,7 +14,9 @@
 # limitations under the License.
 
 import os
+import re
 import sys
+import urlparse
 import xml.dom.minidom
 
 from git_config import GitConfig, IsId
@@ -24,6 +26,9 @@ from error import ManifestParseError
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 
+urlparse.uses_relative.extend(['ssh', 'git'])
+urlparse.uses_netloc.extend(['ssh', 'git'])
+
 class _Default(object):
   """Project defaults within the manifest."""
 
@@ -35,16 +40,26 @@ class _XmlRemote(object):
   def __init__(self,
                name,
                fetch=None,
+               manifestUrl=None,
                review=None):
     self.name = name
     self.fetchUrl = fetch
+    self.manifestUrl = manifestUrl
     self.reviewUrl = review
+    self.resolvedFetchUrl = self._resolveFetchUrl()
+
+  def _resolveFetchUrl(self):
+    url = self.fetchUrl.rstrip('/')
+    manifestUrl = self.manifestUrl.rstrip('/')
+    # urljoin will get confused if there is no scheme in the base url
+    # ie, if manifestUrl is of the form <hostname:port>
+    if manifestUrl.find(':') != manifestUrl.find('/') - 1:
+        manifestUrl = 'gopher://' + manifestUrl
+    url = urlparse.urljoin(manifestUrl, url)
+    return re.sub(r'^gopher://', '', url)
 
   def ToRemoteSpec(self, projectName):
-    url = self.fetchUrl
-    while url.endswith('/'):
-      url = url[:-1]
-    url += '/%s.git' % projectName
+    url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
     return RemoteSpec(self.name, url, self.reviewUrl)
 
 class XmlManifest(object):
@@ -213,7 +228,11 @@ class XmlManifest(object):
   @property
   def manifest_server(self):
     self._Load()
-    return self._manifest_server
+
+    if self._manifest_server:
+      return self._manifest_server
+
+    return self.manifestProject.config.GetString('repo.manifest-server')
 
   @property
   def IsMirror(self):
@@ -357,7 +376,7 @@ class XmlManifest(object):
       raise ManifestParseError, 'refusing to mirror %s' % m_url
 
     if self._default and self._default.remote:
-      url = self._default.remote.fetchUrl
+      url = self._default.remote.resolvedFetchUrl
       if not url.endswith('/'):
         url += '/'
       if m_url.startswith(url):
@@ -366,7 +385,8 @@ class XmlManifest(object):
 
     if name is None:
       s = m_url.rindex('/') + 1
-      remote = _XmlRemote('origin', m_url[:s])
+      manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+      remote = _XmlRemote('origin', m_url[:s], manifestUrl)
       name = m_url[s:]
 
     if name.endswith('.git'):
@@ -394,7 +414,8 @@ class XmlManifest(object):
     review = node.getAttribute('review')
     if review == '':
       review = None
-    return _XmlRemote(name, fetch, review)
+    manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+    return _XmlRemote(name, fetch, manifestUrl, review)
 
   def _ParseDefault(self, node):
     """

project.py

@@ -16,20 +16,36 @@ import traceback
 import errno
 import filecmp
 import os
+import random
 import re
 import shutil
 import stat
 import sys
+import time
 import urllib2
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
+try:
+  from os import SEEK_END
+except ImportError:
+  SEEK_END = 2
+
 from color import Coloring
 from git_command import GitCommand
-from git_config import GitConfig, IsId
+from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from error import DownloadError
 from error import GitError, HookError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
+from progress import Progress
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
+_urllib_lock = _threading.Lock()
+
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -884,32 +900,35 @@ class Project(object):
 
 ## Sync ##
 
-  def Sync_NetworkHalf(self, quiet=False):
+  def Sync_NetworkHalf(self, quiet=False, is_new=None, current_branch_only=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
-    is_new = not self.Exists
+    if is_new is None:
+      is_new = not self.Exists
     if is_new:
-      if not quiet:
-        print >>sys.stderr
-        print >>sys.stderr, 'Initializing project %s ...' % self.name
       self._InitGitDir()
-
     self._InitRemote()
-    if not self._RemoteFetch(initial=is_new, quiet=quiet):
-      return False
 
-    #Check that the requested ref was found after fetch
-    #
-    try:
-      self.GetRevisionId()
-    except ManifestInvalidRevisionError:
-      # if the ref is a tag. We can try fetching
-      # the tag manually as a last resort
-      #
-      rev = self.revisionExpr
-      if rev.startswith(R_TAGS):
-        self._RemoteFetch(None, rev[len(R_TAGS):], quiet=quiet)
+    if is_new:
+      alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      try:
+        fd = open(alt, 'rb')
+        try:
+          alt_dir = fd.readline().rstrip()
+        finally:
+          fd.close()
+      except IOError:
+        alt_dir = None
+    else:
+      alt_dir = None
+
+    if alt_dir is None and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+      is_new = False
+
+    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+                             current_branch_only=current_branch_only):
+      return False
 
     if self.worktree:
       self._InitMRef()
@@ -997,7 +1016,7 @@ class Project(object):
 
     if not branch.LocalMerge:
       # The current branch has no tracking configuration.
-      # Jump off it to a deatched HEAD.
+      # Jump off it to a detached HEAD.
       #
       syncbuf.info(self,
                    "leaving %s; does not track upstream",
@@ -1305,31 +1324,41 @@ class Project(object):
 
 ## Direct Git Commands ##
 
-  def _RemoteFetch(self, name=None, tag=None,
+  def _RemoteFetch(self, name=None,
+                   current_branch_only=False,
                    initial=False,
-                   quiet=False):
+                   quiet=False,
+                   alt_dir=None):
+
+    is_sha1 = False
+    tag_name = None
+
+    if current_branch_only:
+      if ID_RE.match(self.revisionExpr) is not None:
+        is_sha1 = True
+      elif self.revisionExpr.startswith(R_TAGS):
+        # this is a tag and its sha1 value should never change
+        tag_name = self.revisionExpr[len(R_TAGS):]
+
+      if is_sha1 or tag_name is not None:
+        try:
+          self.GetRevisionId()
+          return True
+        except ManifestInvalidRevisionError:
+          # There is no such persistent revision. We have to fetch it.
+          pass
+
     if not name:
       name = self.remote.name
 
     ssh_proxy = False
-    if self.GetRemote(name).PreConnectFetch():
+    remote = self.GetRemote(name)
+    if remote.PreConnectFetch():
       ssh_proxy = True
 
     if initial:
-      alt = os.path.join(self.gitdir, 'objects/info/alternates')
-      try:
-        fd = open(alt, 'rb')
-        try:
-          ref_dir = fd.readline()
-          if ref_dir and ref_dir.endswith('\n'):
-            ref_dir = ref_dir[:-1]
-        finally:
-          fd.close()
-      except IOError, e:
-        ref_dir = None
-
-      if ref_dir and 'objects' == os.path.basename(ref_dir):
-        ref_dir = os.path.dirname(ref_dir)
+      if alt_dir and 'objects' == os.path.basename(alt_dir):
+        ref_dir = os.path.dirname(alt_dir)
         packed_refs = os.path.join(self.gitdir, 'packed-refs')
         remote = self.GetRemote(name)
 
@@ -1365,9 +1394,8 @@ class Project(object):
             old_packed += line
 
         _lwrite(packed_refs, tmp_packed)
-
       else:
-        ref_dir = None
+        alt_dir = None
 
     cmd = ['fetch']
 
@@ -1382,25 +1410,173 @@ class Project(object):
     if not self.worktree:
       cmd.append('--update-head-ok')
     cmd.append(name)
-    if tag is not None:
-      cmd.append('tag')
-      cmd.append(tag)
 
-    ok = GitCommand(self,
-                    cmd,
-                    bare = True,
-                    ssh_proxy = ssh_proxy).Wait() == 0
+    if not current_branch_only or is_sha1:
+      # Fetch whole repo
+      cmd.append('--tags')
+      cmd.append((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))
+    elif tag_name is not None:
+      cmd.append('tag')
+      cmd.append(tag_name)
+    else:
+      branch = self.revisionExpr
+      if branch.startswith(R_HEADS):
+        branch = branch[len(R_HEADS):]
+      cmd.append((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch))
+
+    ok = False
+    for i in range(2):
+      if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0:
+        ok = True
+        break
+      time.sleep(random.randint(30, 45))
 
     if initial:
-      if ref_dir:
+      if alt_dir:
         if old_packed != '':
           _lwrite(packed_refs, old_packed)
         else:
           os.remove(packed_refs)
       self.bare_git.pack_refs('--all', '--prune')
-
     return ok
 
+  def _ApplyCloneBundle(self, initial=False, quiet=False):
+    if initial and self.manifest.manifestProject.config.GetString('repo.depth'):
+      return False
+
+    remote = self.GetRemote(self.remote.name)
+    bundle_url = remote.url + '/clone.bundle'
+    bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+    if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
+      return False
+
+    bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
+    bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
+
+    exist_dst = os.path.exists(bundle_dst)
+    exist_tmp = os.path.exists(bundle_tmp)
+
+    if not initial and not exist_dst and not exist_tmp:
+      return False
+
+    if not exist_dst:
+      exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
+    if not exist_dst:
+      return False
+
+    cmd = ['fetch']
+    if quiet:
+      cmd.append('--quiet')
+    if not self.worktree:
+      cmd.append('--update-head-ok')
+    cmd.append(bundle_dst)
+    for f in remote.fetch:
+      cmd.append(str(f))
+    cmd.append('refs/tags/*:refs/tags/*')
+
+    ok = GitCommand(self, cmd, bare=True).Wait() == 0
+    if os.path.exists(bundle_dst):
+      os.remove(bundle_dst)
+    if os.path.exists(bundle_tmp):
+      os.remove(bundle_tmp)
+    return ok
+
+  def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
+    keep = True
+    done = False
+    dest = open(tmpPath, 'a+b')
+    try:
+      dest.seek(0, SEEK_END)
+      pos = dest.tell()
+
+      _urllib_lock.acquire()
+      try:
+        req = urllib2.Request(srcUrl)
+        if pos > 0:
+          req.add_header('Range', 'bytes=%d-' % pos)
+
+        try:
+          r = urllib2.urlopen(req)
+        except urllib2.HTTPError, e:
+          def _content_type():
+            try:
+              return e.info()['content-type']
+            except:
+              return None
+
+          if e.code == 404:
+            keep = False
+            return False
+          elif _content_type() == 'text/plain':
+            try:
+              msg = e.read()
+              if len(msg) > 0 and msg[-1] == '\n':
+                msg = msg[0:-1]
+              msg = ' (%s)' % msg
+            except:
+              msg = ''
+          else:
+            try:
+              from BaseHTTPServer import BaseHTTPRequestHandler
+              res = BaseHTTPRequestHandler.responses[e.code]
+              msg = ' (%s: %s)' % (res[0], res[1])
+            except:
+              msg = ''
+          raise DownloadError('HTTP %s%s' % (e.code, msg))
+        except urllib2.URLError, e:
+          raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+      finally:
+        _urllib_lock.release()
+
+      p = None
+      try:
+        size = r.headers['content-length']
+        unit = 1 << 10
+
+        if size and not quiet:
+          if size > 1024 * 1.3:
+            unit = 1 << 20
+            desc = 'MB'
+          else:
+            desc = 'KB'
+          p = Progress(
+            'Downloading %s' % self.relpath,
+            int(size) / unit,
+            units=desc)
+          if pos > 0:
+            p.update(pos / unit)
+
+        s = 0
+        while True:
+          d = r.read(8192)
+          if d == '':
+            done = True
+            return True
+          dest.write(d)
+          if p:
+            s += len(d)
+            if s >= unit:
+              p.update(s / unit)
+              s = s % unit
+        if p:
+          if s >= unit:
+            p.update(s / unit)
+          else:
+            p.update(1)
+      finally:
+        r.close()
+        if p:
+          p.end()
+    finally:
+      dest.close()
+
+      if os.path.exists(dstPath):
+        os.remove(dstPath)
+      if done:
+        os.rename(tmpPath, dstPath)
+      elif not keep:
+        os.remove(tmpPath)
+
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
     if quiet:

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 12)
+VERSION = (1, 14)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -91,6 +91,7 @@ import re
 import readline
 import subprocess
 import sys
+import urllib2
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
@@ -138,6 +139,11 @@ group.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+# Other
+group = init_optparse.add_option_group('Other options')
+group.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
 
 class CloneFailure(Exception):
   """Indicate the remote clone of repo itself failed.
@@ -148,7 +154,7 @@ def _Init(args):
   """Installs repo by cloning it over the network.
   """
   opt, args = init_optparse.parse_args(args)
-  if args or not opt.manifest_url:
+  if args:
     init_optparse.print_usage()
     sys.exit(1)
 
@@ -187,10 +193,6 @@ def _Init(args):
     else:
       can_verify = True
 
-    if not opt.quiet:
-      print >>sys.stderr, 'Getting repo ...'
-      print >>sys.stderr, '   from %s' % url
-
     dst = os.path.abspath(os.path.join(repodir, S_repo))
     _Clone(url, dst, opt.quiet)
 
@@ -300,15 +302,42 @@ def _SetConfig(local, name, value):
     raise CloneFailure()
 
 
-def _Fetch(local, quiet, *args):
+def _InitHttp():
+  handlers = []
+
+  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  try:
+    import netrc
+    n = netrc.netrc()
+    for host in n.hosts:
+      p = n.hosts[host]
+      mgr.add_password(None, 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
+  except:
+    pass
+  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+
+  if 'http_proxy' in os.environ:
+    url = os.environ['http_proxy']
+    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+  if 'REPO_CURL_VERBOSE' in os.environ:
+    handlers.append(urllib2.HTTPHandler(debuglevel=1))
+    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+  urllib2.install_opener(urllib2.build_opener(*handlers))
+
+def _Fetch(url, local, src, quiet):
+  if not quiet:
+    print >>sys.stderr, 'Get %s' % url
+
   cmd = [GIT, 'fetch']
   if quiet:
     cmd.append('--quiet')
     err = subprocess.PIPE
   else:
     err = None
-  cmd.extend(args)
-  cmd.append('origin')
+  cmd.append(src)
+  cmd.append('+refs/heads/*:refs/remotes/origin/*')
+  cmd.append('refs/tags/*:refs/tags/*')
 
   proc = subprocess.Popen(cmd, cwd = local, stderr = err)
   if err:
@@ -317,6 +346,62 @@ def _Fetch(local, quiet, *args):
   if proc.wait() != 0:
     raise CloneFailure()
 
+def _DownloadBundle(url, local, quiet):
+  if not url.endswith('/'):
+    url += '/'
+  url += 'clone.bundle'
+
+  proc = subprocess.Popen(
+    [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
+    cwd = local,
+    stdout = subprocess.PIPE)
+  for line in proc.stdout:
+    m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
+    if m:
+      new_url = m.group(1)
+      old_url = m.group(2)
+      if url.startswith(old_url):
+        url = new_url + url[len(old_url):]
+        break
+  proc.stdout.close()
+  proc.wait()
+
+  if not url.startswith('http:') and not url.startswith('https:'):
+    return False
+
+  dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b')
+  try:
+    try:
+      r = urllib2.urlopen(url)
+    except urllib2.HTTPError, e:
+      if e.code == 404:
+        return False
+      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      print >>sys.stderr, 'fatal: HTTP error %s' % e.code
+      raise CloneFailure()
+    except urllib2.URLError, e:
+      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      print >>sys.stderr, 'fatal: error %s' % e.reason
+      raise CloneFailure()
+    try:
+      if not quiet:
+        print >>sys.stderr, 'Get %s' % url
+      while True:
+        buf = r.read(8192)
+        if buf == '':
+          return True
+        dest.write(buf)
+    finally:
+      r.close()
+  finally:
+    dest.close()
+
+def _ImportBundle(local):
+  path = os.path.join(local, '.git', 'clone.bundle')
+  try:
+    _Fetch(local, local, path, True)
+  finally:
+    os.remove(path)
 
 def _Clone(url, local, quiet):
   """Clones a git repository to a new subdirectory of repodir
@@ -344,11 +429,14 @@ def _Clone(url, local, quiet):
     print >>sys.stderr, 'fatal: could not create %s' % local
     raise CloneFailure()
 
+  _InitHttp()
   _SetConfig(local, 'remote.origin.url', url)
   _SetConfig(local, 'remote.origin.fetch',
                     '+refs/heads/*:refs/remotes/origin/*')
-  _Fetch(local, quiet)
-  _Fetch(local, quiet, '--tags')
+  if _DownloadBundle(url, local, quiet):
+    _ImportBundle(local)
+  else:
+    _Fetch(url, local, 'origin', quiet)
 
 
 def _Verify(cwd, branch, quiet):

subcmds/help.py

@@ -165,6 +165,7 @@ See 'repo help --all' for a complete list of recognized commands.
         print >>sys.stderr, "repo: '%s' is not a repo command." % name
         sys.exit(1)
 
+      cmd.manifest = self.manifest
       self._PrintCommandHelp(cmd)
 
     else:

subcmds/init.py

@@ -21,7 +21,9 @@ from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from project import SyncBuffer
+from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION
+from git_config import GitConfig
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
@@ -35,6 +37,20 @@ The latest repo source code and manifest collection is downloaded
 from the server and is installed in the .repo/ directory in the
 current working directory.
 
+The optional -u argument can be used to specify a URL to the
+manifest project. It is also possible to have a git configuration
+section as below to use 'identifier' as argument to -u:
+
+  [repo-manifest "identifier"]
+    url = ...
+    server = ...
+    reference = ...
+
+Only the url is required - the others are optional.
+
+If no -u argument is specified, the 'repo-manifest' section named
+'default' will be used if present.
+
 The optional -b argument can be used to select the manifest branch
 to checkout and use.  If no branch is specified, master is assumed.
 
@@ -68,7 +84,7 @@ to update the working directory files.
     # Manifest
     g = p.add_option_group('Manifest options')
     g.add_option('-u', '--manifest-url',
-                 dest='manifest_url',
+                 dest='manifest_url', default='default',
                  help='manifest repository location', metavar='URL')
     g.add_option('-b', '--manifest-branch',
                  dest='manifest_branch',
@@ -98,18 +114,39 @@ to update the working directory files.
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+    # Other
+    g = p.add_option_group('Other options')
+    g.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
+    manifest_server = None
+
+    # The manifest url may point out a manifest section in the config
+    key = 'repo-manifest.%s.' % opt.manifest_url
+    if GitConfig.ForUser().GetString(key + 'url'):
+      opt.manifest_url = GitConfig.ForUser().GetString(key + 'url')
+
+      # Also copy other options to the manifest config if not specified already.
+      if not opt.reference:
+        opt.reference = GitConfig.ForUser().GetString(key + 'reference')
+      manifest_server = GitConfig.ForUser().GetString(key + 'server')
 
     if is_new:
-      if not opt.manifest_url:
-        print >>sys.stderr, 'fatal: manifest url (-u) is required.'
+      if not opt.manifest_url or opt.manifest_url == 'default':
+        print >>sys.stderr, """\
+fatal: missing manifest url (-u) and no default found.
+
+  tip: The global git configuration key 'repo-manifest.default.url' can
+       be used to specify a default url."""
         sys.exit(1)
 
       if not opt.quiet:
-        print >>sys.stderr, 'Getting manifest ...'
-        print >>sys.stderr, '   from %s' % opt.manifest_url
+        print >>sys.stderr, 'Get %s' \
+          % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
       m._InitGitDir()
 
       if opt.manifest_branch:
@@ -128,6 +165,9 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
+    if manifest_server:
+      m.config.SetString('repo.manifest-server', manifest_server)
+
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 
@@ -138,7 +178,7 @@ to update the working directory files.
         print >>sys.stderr, 'fatal: --mirror not supported on existing client'
         sys.exit(1)
 
-    if not m.Sync_NetworkHalf():
+    if not m.Sync_NetworkHalf(is_new=is_new):
       r = m.GetRemote(m.remote.name)
       print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
 
@@ -178,6 +218,24 @@ to update the working directory files.
       return value
     return a
 
+  def _ShouldConfigureUser(self):
+    gc = self.manifest.globalConfig
+    mp = self.manifest.manifestProject
+
+    # If we don't have local settings, get from global.
+    if not mp.config.Has('user.name') or not mp.config.Has('user.email'):
+      if not gc.Has('user.name') or not gc.Has('user.email'):
+        return True
+
+      mp.config.SetString('user.name', gc.GetString('user.name'))
+      mp.config.SetString('user.email', gc.GetString('user.email'))
+
+    print ''
+    print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+                                         mp.config.GetString('user.email'))
+    print 'If you want to change this, please re-run \'repo init\' with --config-name'
+    return False
+
   def _ConfigureUser(self):
     mp = self.manifest.manifestProject
 
@@ -188,7 +246,7 @@ to update the working directory files.
 
       print ''
       print 'Your identity is: %s <%s>' % (name, email)
-      sys.stdout.write('is this correct [y/n]? ')
+      sys.stdout.write('is this correct [y/N]? ')
       a = sys.stdin.readline().strip()
       if a in ('yes', 'y', 't', 'true'):
         break
@@ -230,7 +288,7 @@ to update the working directory files.
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
 
-    sys.stdout.write('Enable color display in this user account (y/n)? ')
+    sys.stdout.write('Enable color display in this user account (y/N)? ')
     a = sys.stdin.readline().strip().lower()
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
@@ -260,7 +318,8 @@ to update the working directory files.
     self._LinkManifest(opt.manifest_name)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-      self._ConfigureUser()
+      if opt.config_name or self._ShouldConfigureUser():
+        self._ConfigureUser()
       self._ConfigureColor()
 
     self._ConfigureDepth(opt)

subcmds/sync.py

@@ -131,12 +131,14 @@ later is required to fix a server side protocol bug.
     p.add_option('-d','--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
+    p.add_option('-c','--current-branch',
+                 dest='current_branch_only', action='store_true',
+                 help='fetch only current branch from server')
     p.add_option('-q','--quiet',
                  dest='quiet', action='store_true',
                  help='be more quiet')
     p.add_option('-j','--jobs',
                  dest='jobs', action='store', type='int',
-                 default=self.jobs,
                  help="projects to fetch simultaneously (default %d)" % self.jobs)
     if show_smart:
       p.add_option('-s', '--smart-sync',
@@ -180,7 +182,8 @@ later is required to fix a server side protocol bug.
       # - We always make sure we unlock the lock if we locked it.
       try:
         try:
-          success = project.Sync_NetworkHalf(quiet=opt.quiet)
+          success = project.Sync_NetworkHalf(quiet=opt.quiet,
+                                             current_branch_only=opt.current_branch_only)
 
           # Lock around all the rest of the code, since printing, updating a set
           # and Progress.update() are not thread safe.
@@ -196,15 +199,11 @@ later is required to fix a server side protocol bug.
 
           fetched.add(project.gitdir)
           pm.update()
-        except BaseException, e:
-          # Notify the _Fetch() function about all errors.
+        except _FetchError:
           err_event.set()
-
-          # If we got our own _FetchError, we don't want a stack trace.
-          # However, if we got something else (something in Sync_NetworkHalf?),
-          # we'd like one (so re-raise after we've set err_event).
-          if not isinstance(e, _FetchError):
-            raise
+        except:
+          err_event.set()
+          raise
       finally:
         if did_lock:
           lock.release()
@@ -217,7 +216,8 @@ later is required to fix a server side protocol bug.
     if self.jobs == 1:
       for project in projects:
         pm.update()
-        if project.Sync_NetworkHalf(quiet=opt.quiet):
+        if project.Sync_NetworkHalf(quiet=opt.quiet,
+                                    current_branch_only=opt.current_branch_only):
           fetched.add(project.gitdir)
         else:
           print >>sys.stderr, 'error: Cannot fetch %s' % project.name
@@ -393,7 +393,8 @@ uncommitted changes are present' % project.relpath
       _PostRepoUpgrade(self.manifest)
 
     if not opt.local_only:
-      mp.Sync_NetworkHalf(quiet=opt.quiet)
+      mp.Sync_NetworkHalf(quiet=opt.quiet,
+                          current_branch_only=opt.current_branch_only)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
@@ -401,6 +402,8 @@ uncommitted changes are present' % project.relpath
       if not syncbuf.Finish():
         sys.exit(1)
       self.manifest._Unload()
+      if opt.jobs is None:
+        self.jobs = self.manifest.default.sync_j
     all = self.GetProjects(args, missing_ok=True)
 
     if not opt.local_only:

subcmds/upload.py

@@ -73,7 +73,7 @@ Configuration
 
 review.URL.autoupload:
 
-To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+To disable the "Upload ... (y/N)?" prompt, you can set a per-project
 or global Git configuration option.  If review.URL.autoupload is set
 to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
@@ -162,7 +162,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       date = branch.date
       list = branch.commits
 
-      print 'Upload project %s/:' % project.relpath
+      print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
       print '  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(list),
@@ -171,7 +171,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       for commit in list:
         print '         %s' % commit
 
-      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      sys.stdout.write('to %s (y/N)? ' % remote.review)
       answer = sys.stdin.readline().strip()
       answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
 
@@ -202,11 +202,12 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
         if b:
           script.append('#')
-        script.append('#  branch %s (%2d commit%s, %s):' % (
+        script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
                       len(list),
                       len(list) != 1 and 's' or '',
-                      date))
+                      date,
+                      project.revisionExpr))
         for commit in list:
           script.append('#         %s' % commit)
         b[name] = branch
@@ -215,6 +216,11 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       branches[project.name] = b
     script.append('')
 
+    script = [ x.encode('utf-8')
+             if issubclass(type(x), unicode)
+             else x
+             for x in script ]
+
     script = Editor.EditString("\n".join(script)).split("\n")
 
     project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
@@ -294,7 +300,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
             # if they want to auto upload, let's not ask because it could be automated
             if answer is None:
-                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ')
+                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
                 a = sys.stdin.readline().strip().lower()
                 if a not in ('y', 'yes', 't', 'true', 'on'):
                     print >>sys.stderr, "skipping upload"