upload: Support uploading to Gerrit over https://

If SSH is not available, Gerrit returns NOT_AVAILABLE to the /ssh_info
query made by repo upload. In this case fallback to the /p/$PROJECT URL
that Gerrit also exports and use that for uploads.

Change-Id: I1e3e39ab709ecc0a692614a41a42446426f39c08

error.py

@@ -57,6 +57,15 @@ class UploadError(Exception):
   def __str__(self):
     return self.reason
 
+class DownloadError(Exception):
+  """Cannot download a repository.
+  """
+  def __init__(self, reason):
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
 class NoSuchProjectError(Exception):
   """A specified project does not exist in the work tree.
   """

git_command.py

@@ -221,26 +221,10 @@ class GitCommand(object):
     self.stdin = p.stdin
 
   def Wait(self):
-    p = self.process
-
-    if p.stdin:
-      p.stdin.close()
-      self.stdin = None
-
-    if p.stdout:
-      self.stdout = p.stdout.read()
-      p.stdout.close()
-    else:
-      p.stdout = None
-
-    if p.stderr:
-      self.stderr = p.stderr.read()
-      p.stderr.close()
-    else:
-      p.stderr = None
-
     try:
-      rc = p.wait()
+      p = self.process
+      (self.stdout, self.stderr) = p.communicate()
+      rc = p.returncode
     finally:
       _remove_ssh_client(p)
     return rc

git_config.py

@@ -26,7 +26,6 @@ import time
 import urllib2
 
 from signal import SIGTERM
-from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
 
@@ -491,6 +490,12 @@ def close_ssh():
 URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
 URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
 
+def GetSchemeFromUrl(url):
+  m = URI_ALL.match(url)
+  if m:
+    return m.group(1)
+  return None
+
 def _preconnect(url):
   m = URI_ALL.match(url)
   if m:
@@ -522,7 +527,7 @@ class Remote(object):
     self.projectname = self._Get('projectname')
     self.fetch = map(lambda x: RefSpec.FromString(x),
                      self._Get('fetch', all=True))
-    self._review_protocol = None
+    self._review_url = None
 
   def _InsteadOf(self):
     globCfg = GitConfig.ForUser()
@@ -549,9 +554,8 @@ class Remote(object):
     connectionUrl = self._InsteadOf()
     return _preconnect(connectionUrl)
 
-  @property
-  def ReviewProtocol(self):
-    if self._review_protocol is None:
+  def ReviewUrl(self, userEmail):
+    if self._review_url is None:
       if self.review is None:
         return None
 
@@ -560,57 +564,47 @@ class Remote(object):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
-      if not u.endswith('/ssh_info'):
-        if not u.endswith('/'):
-          u += '/'
-        u += 'ssh_info'
+      if u.endswith('/ssh_info'):
+        u = u[:len(u) - len('/ssh_info')]
+      if not u.endswith('/'):
+         u += '/'
+      http_url = u
 
       if u in REVIEW_CACHE:
-        info = REVIEW_CACHE[u]
-        self._review_protocol = info[0]
-        self._review_host = info[1]
-        self._review_port = info[2]
+        self._review_url = REVIEW_CACHE[u]
+      elif 'REPO_HOST_PORT_INFO' in os.environ:
+        host, port = os.environ['REPO_HOST_PORT_INFO'].split()
+        self._review_url = self._SshReviewUrl(userEmail, host, port)
+        REVIEW_CACHE[u] = self._review_url
       else:
         try:
-          info = urlopen(u).read()
-          if info == 'NOT_AVAILABLE':
-            raise UploadError('%s: SSH disabled' % self.review)
+          info_url = u + 'ssh_info'
+          info = urllib2.urlopen(info_url).read()
           if '<' in info:
             # Assume the server gave us some sort of HTML
             # response back, like maybe a login page.
             #
-            raise UploadError('%s: Cannot parse response' % u)
+            raise UploadError('%s: Cannot parse response' % info_url)
 
-          self._review_protocol = 'ssh'
-          self._review_host = info.split(" ")[0]
-          self._review_port = info.split(" ")[1]
-        except urllib2.URLError, e:
-          raise UploadError('%s: %s' % (self.review, e.reason[1]))
-        except HTTPError, e:
-          if e.code == 404:
-            self._review_protocol = 'http-post'
-            self._review_host = None
-            self._review_port = None
+          if info == 'NOT_AVAILABLE':
+            # Assume HTTP if SSH is not enabled.
+            self._review_url = http_url + 'p/'
           else:
-            raise UploadError('Upload over ssh unavailable')
+            host, port = info.split()
+            self._review_url = self._SshReviewUrl(userEmail, host, port)
+        except urllib2.HTTPError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
+        except urllib2.URLError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
 
-        REVIEW_CACHE[u] = (
-          self._review_protocol,
-          self._review_host,
-          self._review_port)
-    return self._review_protocol
+        REVIEW_CACHE[u] = self._review_url
+    return self._review_url + self.projectname
 
-  def SshReviewUrl(self, userEmail):
-    if self.ReviewProtocol != 'ssh':
-      return None
+  def _SshReviewUrl(self, userEmail, host, port):
     username = self._config.GetString('review.%s.username' % self.review)
     if username is None:
-      username = userEmail.split("@")[0]
-    return 'ssh://%s@%s:%s/%s' % (
-      username,
-      self._review_host,
-      self._review_port,
-      self.projectname)
+      username = userEmail.split('@')[0]
+    return 'ssh://%s@%s:%s/' % (username, host, port)
 
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.

git_refs.py

@@ -139,13 +139,15 @@ class GitRefs(object):
   def _ReadLoose1(self, path, name):
     try:
       fd = open(path, 'rb')
-      mtime = os.path.getmtime(path)
-    except OSError:
-      return
-    except IOError:
+    except:
       return
+
     try:
-      id = fd.readline()
+      try:
+        mtime = os.path.getmtime(path)
+        id = fd.readline()
+      except:
+        return
     finally:
       fd.close()
 

main.py

@@ -36,7 +36,9 @@ from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
+from subcmds.version import Version
 from editor import Editor
+from error import DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoSuchProjectError
 from error import RepoChangedException
@@ -72,6 +74,7 @@ class _Repo(object):
     all_commands['branch'] = all_commands['branches']
 
   def _Run(self, argv):
+    result = 0
     name = None
     glob = []
 
@@ -95,7 +98,7 @@ class _Repo(object):
         name = 'version'
       else:
         print >>sys.stderr, 'fatal: invalid usage of --version'
-        sys.exit(1)
+        return 1
 
     try:
       cmd = self.commands[name]
@@ -103,7 +106,7 @@ class _Repo(object):
       print >>sys.stderr,\
             "repo: '%s' is not a repo command.  See 'repo help'."\
             % name
-      sys.exit(1)
+      return 1
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
@@ -113,7 +116,7 @@ class _Repo(object):
       print >>sys.stderr, \
             "fatal: '%s' requires a working directory"\
             % name
-      sys.exit(1)
+      return 1
 
     copts, cargs = cmd.OptionParser.parse_args(argv)
 
@@ -131,7 +134,7 @@ class _Repo(object):
     try:
       start = time.time()
       try:
-        cmd.Execute(copts, cargs)
+        result = cmd.Execute(copts, cargs)
       finally:
         elapsed = time.time() - start
         hours, remainder = divmod(elapsed, 3600)
@@ -143,15 +146,20 @@ class _Repo(object):
           else:
             print >>sys.stderr, 'real\t%dh%dm%.3fs' \
               % (hours, minutes, seconds)
+    except DownloadError, e:
+      print >>sys.stderr, 'error: %s' % str(e)
+      return 1
     except ManifestInvalidRevisionError, e:
       print >>sys.stderr, 'error: %s' % str(e)
-      sys.exit(1)
+      return 1
     except NoSuchProjectError, e:
       if e.name:
         print >>sys.stderr, 'error: project %s not found' % e.name
       else:
         print >>sys.stderr, 'error: no project in current directory'
-      sys.exit(1)
+      return 1
+
+    return result
 
 def _MyRepoPath():
   return os.path.dirname(__file__)
@@ -269,6 +277,24 @@ class _UserAgentHandler(urllib2.BaseHandler):
     req.add_header('User-Agent', _UserAgent())
     return req
 
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_auth_reqed(self, authreq, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+        self, authreq, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -281,7 +307,9 @@ def init_http():
       mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
   except netrc.NetrcParseError:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  except IOError:
+    pass
+  handlers.append(_BasicAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
@@ -292,6 +320,8 @@ def init_http():
   urllib2.install_opener(urllib2.build_opener(*handlers))
 
 def _Main(argv):
+  result = 0
+
   opt = optparse.OptionParser(usage="repo wrapperinfo -- ...")
   opt.add_option("--repo-dir", dest="repodir",
                  help="path to .repo/")
@@ -305,16 +335,19 @@ def _Main(argv):
   _CheckWrapperVersion(opt.wrapper_version, opt.wrapper_path)
   _CheckRepoDir(opt.repodir)
 
+  Version.wrapper_version = opt.wrapper_version
+  Version.wrapper_path = opt.wrapper_path
+
   repo = _Repo(opt.repodir)
   try:
     try:
       init_ssh()
       init_http()
-      repo._Run(argv)
+      result = repo._Run(argv) or 0
     finally:
       close_ssh()
   except KeyboardInterrupt:
-    sys.exit(1)
+    result = 1
   except RepoChangedException, rce:
     # If repo changed, re-exec ourselves.
     #
@@ -325,7 +358,9 @@ def _Main(argv):
     except OSError, e:
       print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
       print >>sys.stderr, 'fatal: %s' % e
-      sys.exit(128)
+      result = 128
+
+  sys.exit(result)
 
 if __name__ == '__main__':
   _Main(sys.argv[1:])

manifest_xml.py

@@ -14,7 +14,9 @@
 # limitations under the License.
 
 import os
+import re
 import sys
+import urlparse
 import xml.dom.minidom
 
 from git_config import GitConfig, IsId
@@ -24,6 +26,9 @@ from error import ManifestParseError
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 
+urlparse.uses_relative.extend(['ssh', 'git'])
+urlparse.uses_netloc.extend(['ssh', 'git'])
+
 class _Default(object):
   """Project defaults within the manifest."""
 
@@ -35,16 +40,26 @@ class _XmlRemote(object):
   def __init__(self,
                name,
                fetch=None,
+               manifestUrl=None,
                review=None):
     self.name = name
     self.fetchUrl = fetch
+    self.manifestUrl = manifestUrl
     self.reviewUrl = review
+    self.resolvedFetchUrl = self._resolveFetchUrl()
+
+  def _resolveFetchUrl(self):
+    url = self.fetchUrl.rstrip('/')
+    manifestUrl = self.manifestUrl.rstrip('/')
+    # urljoin will get confused if there is no scheme in the base url
+    # ie, if manifestUrl is of the form <hostname:port>
+    if manifestUrl.find(':') != manifestUrl.find('/') - 1:
+        manifestUrl = 'gopher://' + manifestUrl
+    url = urlparse.urljoin(manifestUrl, url)
+    return re.sub(r'^gopher://', '', url)
 
   def ToRemoteSpec(self, projectName):
-    url = self.fetchUrl
-    while url.endswith('/'):
-      url = url[:-1]
-    url += '/%s.git' % projectName
+    url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
     return RemoteSpec(self.name, url, self.reviewUrl)
 
 class XmlManifest(object):
@@ -357,7 +372,7 @@ class XmlManifest(object):
       raise ManifestParseError, 'refusing to mirror %s' % m_url
 
     if self._default and self._default.remote:
-      url = self._default.remote.fetchUrl
+      url = self._default.remote.resolvedFetchUrl
       if not url.endswith('/'):
         url += '/'
       if m_url.startswith(url):
@@ -366,7 +381,8 @@ class XmlManifest(object):
 
     if name is None:
       s = m_url.rindex('/') + 1
-      remote = _XmlRemote('origin', m_url[:s])
+      manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+      remote = _XmlRemote('origin', m_url[:s], manifestUrl)
       name = m_url[s:]
 
     if name.endswith('.git'):
@@ -394,7 +410,8 @@ class XmlManifest(object):
     review = node.getAttribute('review')
     if review == '':
       review = None
-    return _XmlRemote(name, fetch, review)
+    manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+    return _XmlRemote(name, fetch, manifestUrl, review)
 
   def _ParseDefault(self, node):
     """

project.py

@@ -16,20 +16,36 @@ import traceback
 import errno
 import filecmp
 import os
+import random
 import re
 import shutil
 import stat
 import sys
+import time
 import urllib2
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
+try:
+  from os import SEEK_END
+except ImportError:
+  SEEK_END = 2
+
 from color import Coloring
 from git_command import GitCommand
-from git_config import GitConfig, IsId
+from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from error import DownloadError
 from error import GitError, HookError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
+from progress import Progress
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
+_urllib_lock = _threading.Lock()
+
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -850,31 +866,30 @@ class Project(object):
       branch.remote.projectname = self.name
       branch.remote.Save()
 
-    if branch.remote.ReviewProtocol == 'ssh':
-      if dest_branch.startswith(R_HEADS):
-        dest_branch = dest_branch[len(R_HEADS):]
+    url = branch.remote.ReviewUrl(self.UserEmail)
+    if url is None:
+      raise UploadError('review not configured')
+    cmd = ['push']
 
+    if url.startswith('ssh://'):
       rp = ['gerrit receive-pack']
       for e in people[0]:
         rp.append('--reviewer=%s' % sq(e))
       for e in people[1]:
         rp.append('--cc=%s' % sq(e))
-
-      ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
-      if auto_topic:
-        ref_spec = ref_spec + '/' + branch.name
-
-      cmd = ['push']
       cmd.append('--receive-pack=%s' % " ".join(rp))
-      cmd.append(branch.remote.SshReviewUrl(self.UserEmail))
-      cmd.append(ref_spec)
 
-      if GitCommand(self, cmd, bare = True).Wait() != 0:
-        raise UploadError('Upload failed')
+    cmd.append(url)
 
-    else:
-        raise UploadError('Unsupported protocol %s' \
-          % branch.remote.review)
+    if dest_branch.startswith(R_HEADS):
+      dest_branch = dest_branch[len(R_HEADS):]
+    ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
+    if auto_topic:
+      ref_spec = ref_spec + '/' + branch.name
+    cmd.append(ref_spec)
+
+    if GitCommand(self, cmd, bare = True).Wait() != 0:
+      raise UploadError('Upload failed')
 
     msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
     self.bare_git.UpdateRef(R_PUB + branch.name,
@@ -884,32 +899,35 @@ class Project(object):
 
 ## Sync ##
 
-  def Sync_NetworkHalf(self, quiet=False):
+  def Sync_NetworkHalf(self, quiet=False, is_new=None, current_branch_only=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
-    is_new = not self.Exists
+    if is_new is None:
+      is_new = not self.Exists
     if is_new:
-      if not quiet:
-        print >>sys.stderr
-        print >>sys.stderr, 'Initializing project %s ...' % self.name
       self._InitGitDir()
-
     self._InitRemote()
-    if not self._RemoteFetch(initial=is_new, quiet=quiet):
-      return False
 
-    #Check that the requested ref was found after fetch
-    #
-    try:
-      self.GetRevisionId()
-    except ManifestInvalidRevisionError:
-      # if the ref is a tag. We can try fetching
-      # the tag manually as a last resort
-      #
-      rev = self.revisionExpr
-      if rev.startswith(R_TAGS):
-        self._RemoteFetch(None, rev[len(R_TAGS):], quiet=quiet)
+    if is_new:
+      alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      try:
+        fd = open(alt, 'rb')
+        try:
+          alt_dir = fd.readline().rstrip()
+        finally:
+          fd.close()
+      except IOError:
+        alt_dir = None
+    else:
+      alt_dir = None
+
+    if alt_dir is None and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+      is_new = False
+
+    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+                             current_branch_only=current_branch_only):
+      return False
 
     if self.worktree:
       self._InitMRef()
@@ -997,7 +1015,7 @@ class Project(object):
 
     if not branch.LocalMerge:
       # The current branch has no tracking configuration.
-      # Jump off it to a deatched HEAD.
+      # Jump off it to a detached HEAD.
       #
       syncbuf.info(self,
                    "leaving %s; does not track upstream",
@@ -1305,31 +1323,41 @@ class Project(object):
 
 ## Direct Git Commands ##
 
-  def _RemoteFetch(self, name=None, tag=None,
+  def _RemoteFetch(self, name=None,
+                   current_branch_only=False,
                    initial=False,
-                   quiet=False):
+                   quiet=False,
+                   alt_dir=None):
+
+    is_sha1 = False
+    tag_name = None
+
+    if current_branch_only:
+      if ID_RE.match(self.revisionExpr) is not None:
+        is_sha1 = True
+      elif self.revisionExpr.startswith(R_TAGS):
+        # this is a tag and its sha1 value should never change
+        tag_name = self.revisionExpr[len(R_TAGS):]
+
+      if is_sha1 or tag_name is not None:
+        try:
+          self.GetRevisionId()
+          return True
+        except ManifestInvalidRevisionError:
+          # There is no such persistent revision. We have to fetch it.
+          pass
+
     if not name:
       name = self.remote.name
 
     ssh_proxy = False
-    if self.GetRemote(name).PreConnectFetch():
+    remote = self.GetRemote(name)
+    if remote.PreConnectFetch():
       ssh_proxy = True
 
     if initial:
-      alt = os.path.join(self.gitdir, 'objects/info/alternates')
-      try:
-        fd = open(alt, 'rb')
-        try:
-          ref_dir = fd.readline()
-          if ref_dir and ref_dir.endswith('\n'):
-            ref_dir = ref_dir[:-1]
-        finally:
-          fd.close()
-      except IOError, e:
-        ref_dir = None
-
-      if ref_dir and 'objects' == os.path.basename(ref_dir):
-        ref_dir = os.path.dirname(ref_dir)
+      if alt_dir and 'objects' == os.path.basename(alt_dir):
+        ref_dir = os.path.dirname(alt_dir)
         packed_refs = os.path.join(self.gitdir, 'packed-refs')
         remote = self.GetRemote(name)
 
@@ -1365,9 +1393,8 @@ class Project(object):
             old_packed += line
 
         _lwrite(packed_refs, tmp_packed)
-
       else:
-        ref_dir = None
+        alt_dir = None
 
     cmd = ['fetch']
 
@@ -1382,25 +1409,173 @@ class Project(object):
     if not self.worktree:
       cmd.append('--update-head-ok')
     cmd.append(name)
-    if tag is not None:
-      cmd.append('tag')
-      cmd.append(tag)
 
-    ok = GitCommand(self,
-                    cmd,
-                    bare = True,
-                    ssh_proxy = ssh_proxy).Wait() == 0
+    if not current_branch_only or is_sha1:
+      # Fetch whole repo
+      cmd.append('--tags')
+      cmd.append((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))
+    elif tag_name is not None:
+      cmd.append('tag')
+      cmd.append(tag_name)
+    else:
+      branch = self.revisionExpr
+      if branch.startswith(R_HEADS):
+        branch = branch[len(R_HEADS):]
+      cmd.append((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch))
+
+    ok = False
+    for i in range(2):
+      if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0:
+        ok = True
+        break
+      time.sleep(random.randint(30, 45))
 
     if initial:
-      if ref_dir:
+      if alt_dir:
         if old_packed != '':
           _lwrite(packed_refs, old_packed)
         else:
           os.remove(packed_refs)
       self.bare_git.pack_refs('--all', '--prune')
-
     return ok
 
+  def _ApplyCloneBundle(self, initial=False, quiet=False):
+    if initial and self.manifest.manifestProject.config.GetString('repo.depth'):
+      return False
+
+    remote = self.GetRemote(self.remote.name)
+    bundle_url = remote.url + '/clone.bundle'
+    bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+    if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
+      return False
+
+    bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
+    bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
+
+    exist_dst = os.path.exists(bundle_dst)
+    exist_tmp = os.path.exists(bundle_tmp)
+
+    if not initial and not exist_dst and not exist_tmp:
+      return False
+
+    if not exist_dst:
+      exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
+    if not exist_dst:
+      return False
+
+    cmd = ['fetch']
+    if quiet:
+      cmd.append('--quiet')
+    if not self.worktree:
+      cmd.append('--update-head-ok')
+    cmd.append(bundle_dst)
+    for f in remote.fetch:
+      cmd.append(str(f))
+    cmd.append('refs/tags/*:refs/tags/*')
+
+    ok = GitCommand(self, cmd, bare=True).Wait() == 0
+    if os.path.exists(bundle_dst):
+      os.remove(bundle_dst)
+    if os.path.exists(bundle_tmp):
+      os.remove(bundle_tmp)
+    return ok
+
+  def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
+    keep = True
+    done = False
+    dest = open(tmpPath, 'a+b')
+    try:
+      dest.seek(0, SEEK_END)
+      pos = dest.tell()
+
+      _urllib_lock.acquire()
+      try:
+        req = urllib2.Request(srcUrl)
+        if pos > 0:
+          req.add_header('Range', 'bytes=%d-' % pos)
+
+        try:
+          r = urllib2.urlopen(req)
+        except urllib2.HTTPError, e:
+          def _content_type():
+            try:
+              return e.info()['content-type']
+            except:
+              return None
+
+          if e.code == 404:
+            keep = False
+            return False
+          elif _content_type() == 'text/plain':
+            try:
+              msg = e.read()
+              if len(msg) > 0 and msg[-1] == '\n':
+                msg = msg[0:-1]
+              msg = ' (%s)' % msg
+            except:
+              msg = ''
+          else:
+            try:
+              from BaseHTTPServer import BaseHTTPRequestHandler
+              res = BaseHTTPRequestHandler.responses[e.code]
+              msg = ' (%s: %s)' % (res[0], res[1])
+            except:
+              msg = ''
+          raise DownloadError('HTTP %s%s' % (e.code, msg))
+        except urllib2.URLError, e:
+          raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+      finally:
+        _urllib_lock.release()
+
+      p = None
+      try:
+        size = r.headers['content-length']
+        unit = 1 << 10
+
+        if size and not quiet:
+          if size > 1024 * 1.3:
+            unit = 1 << 20
+            desc = 'MB'
+          else:
+            desc = 'KB'
+          p = Progress(
+            'Downloading %s' % self.relpath,
+            int(size) / unit,
+            units=desc)
+          if pos > 0:
+            p.update(pos / unit)
+
+        s = 0
+        while True:
+          d = r.read(8192)
+          if d == '':
+            done = True
+            return True
+          dest.write(d)
+          if p:
+            s += len(d)
+            if s >= unit:
+              p.update(s / unit)
+              s = s % unit
+        if p:
+          if s >= unit:
+            p.update(s / unit)
+          else:
+            p.update(1)
+      finally:
+        r.close()
+        if p:
+          p.end()
+    finally:
+      dest.close()
+
+      if os.path.exists(dstPath):
+        os.remove(dstPath)
+      if done:
+        os.rename(tmpPath, dstPath)
+      elif not keep:
+        os.remove(tmpPath)
+
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
     if quiet:

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 12)
+VERSION = (1, 13)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -91,6 +91,7 @@ import re
 import readline
 import subprocess
 import sys
+import urllib2
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
@@ -138,6 +139,11 @@ group.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+# Other
+group = init_optparse.add_option_group('Other options')
+group.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
 
 class CloneFailure(Exception):
   """Indicate the remote clone of repo itself failed.
@@ -187,10 +193,6 @@ def _Init(args):
     else:
       can_verify = True
 
-    if not opt.quiet:
-      print >>sys.stderr, 'Getting repo ...'
-      print >>sys.stderr, '   from %s' % url
-
     dst = os.path.abspath(os.path.join(repodir, S_repo))
     _Clone(url, dst, opt.quiet)
 
@@ -300,15 +302,42 @@ def _SetConfig(local, name, value):
     raise CloneFailure()
 
 
-def _Fetch(local, quiet, *args):
+def _InitHttp():
+  handlers = []
+
+  mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  try:
+    import netrc
+    n = netrc.netrc()
+    for host in n.hosts:
+      p = n.hosts[host]
+      mgr.add_password(None, 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
+  except:
+    pass
+  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+
+  if 'http_proxy' in os.environ:
+    url = os.environ['http_proxy']
+    handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+  if 'REPO_CURL_VERBOSE' in os.environ:
+    handlers.append(urllib2.HTTPHandler(debuglevel=1))
+    handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+  urllib2.install_opener(urllib2.build_opener(*handlers))
+
+def _Fetch(url, local, src, quiet):
+  if not quiet:
+    print >>sys.stderr, 'Get %s' % url
+
   cmd = [GIT, 'fetch']
   if quiet:
     cmd.append('--quiet')
     err = subprocess.PIPE
   else:
     err = None
-  cmd.extend(args)
-  cmd.append('origin')
+  cmd.append(src)
+  cmd.append('+refs/heads/*:refs/remotes/origin/*')
+  cmd.append('refs/tags/*:refs/tags/*')
 
   proc = subprocess.Popen(cmd, cwd = local, stderr = err)
   if err:
@@ -317,6 +346,62 @@ def _Fetch(local, quiet, *args):
   if proc.wait() != 0:
     raise CloneFailure()
 
+def _DownloadBundle(url, local, quiet):
+  if not url.endswith('/'):
+    url += '/'
+  url += 'clone.bundle'
+
+  proc = subprocess.Popen(
+    [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
+    cwd = local,
+    stdout = subprocess.PIPE)
+  for line in proc.stdout:
+    m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
+    if m:
+      new_url = m.group(1)
+      old_url = m.group(2)
+      if url.startswith(old_url):
+        url = new_url + url[len(old_url):]
+        break
+  proc.stdout.close()
+  proc.wait()
+
+  if not url.startswith('http:') and not url.startswith('https:'):
+    return False
+
+  dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b')
+  try:
+    try:
+      r = urllib2.urlopen(url)
+    except urllib2.HTTPError, e:
+      if e.code == 404:
+        return False
+      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      print >>sys.stderr, 'fatal: HTTP error %s' % e.code
+      raise CloneFailure()
+    except urllib2.URLError, e:
+      print >>sys.stderr, 'fatal: Cannot get %s' % url
+      print >>sys.stderr, 'fatal: error %s' % e.reason
+      raise CloneFailure()
+    try:
+      if not quiet:
+        print >>sys.stderr, 'Get %s' % url
+      while True:
+        buf = r.read(8192)
+        if buf == '':
+          return True
+        dest.write(buf)
+    finally:
+      r.close()
+  finally:
+    dest.close()
+
+def _ImportBundle(local):
+  path = os.path.join(local, '.git', 'clone.bundle')
+  try:
+    _Fetch(local, local, path, True)
+  finally:
+    os.remove(path)
 
 def _Clone(url, local, quiet):
   """Clones a git repository to a new subdirectory of repodir
@@ -344,11 +429,14 @@ def _Clone(url, local, quiet):
     print >>sys.stderr, 'fatal: could not create %s' % local
     raise CloneFailure()
 
+  _InitHttp()
   _SetConfig(local, 'remote.origin.url', url)
   _SetConfig(local, 'remote.origin.fetch',
                     '+refs/heads/*:refs/remotes/origin/*')
-  _Fetch(local, quiet)
-  _Fetch(local, quiet, '--tags')
+  if _DownloadBundle(url, local, quiet):
+    _ImportBundle(local)
+  else:
+    _Fetch(url, local, 'origin', quiet)
 
 
 def _Verify(cwd, branch, quiet):

subcmds/help.py

@@ -165,6 +165,7 @@ See 'repo help --all' for a complete list of recognized commands.
         print >>sys.stderr, "repo: '%s' is not a repo command." % name
         sys.exit(1)
 
+      cmd.manifest = self.manifest
       self._PrintCommandHelp(cmd)
 
     else:

subcmds/init.py

@@ -21,6 +21,7 @@ from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from project import SyncBuffer
+from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION
 
 class Init(InteractiveCommand, MirrorSafeCommand):
@@ -98,6 +99,12 @@ to update the working directory files.
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
 
+    # Other
+    g = p.add_option_group('Other options')
+    g.add_option('--config-name',
+                 dest='config_name', action="store_true", default=False,
+                 help='Always prompt for name/e-mail')
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
@@ -108,8 +115,8 @@ to update the working directory files.
         sys.exit(1)
 
       if not opt.quiet:
-        print >>sys.stderr, 'Getting manifest ...'
-        print >>sys.stderr, '   from %s' % opt.manifest_url
+        print >>sys.stderr, 'Get %s' \
+          % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
       m._InitGitDir()
 
       if opt.manifest_branch:
@@ -138,7 +145,7 @@ to update the working directory files.
         print >>sys.stderr, 'fatal: --mirror not supported on existing client'
         sys.exit(1)
 
-    if not m.Sync_NetworkHalf():
+    if not m.Sync_NetworkHalf(is_new=is_new):
       r = m.GetRemote(m.remote.name)
       print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
 
@@ -178,6 +185,24 @@ to update the working directory files.
       return value
     return a
 
+  def _ShouldConfigureUser(self):
+    gc = self.manifest.globalConfig
+    mp = self.manifest.manifestProject
+
+    # If we don't have local settings, get from global.
+    if not mp.config.Has('user.name') or not mp.config.Has('user.email'):
+      if not gc.Has('user.name') or not gc.Has('user.email'):
+        return True
+
+      mp.config.SetString('user.name', gc.GetString('user.name'))
+      mp.config.SetString('user.email', gc.GetString('user.email'))
+
+    print ''
+    print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+                                         mp.config.GetString('user.email'))
+    print 'If you want to change this, please re-run \'repo init\' with --config-name'
+    return False
+
   def _ConfigureUser(self):
     mp = self.manifest.manifestProject
 
@@ -188,7 +213,7 @@ to update the working directory files.
 
       print ''
       print 'Your identity is: %s <%s>' % (name, email)
-      sys.stdout.write('is this correct [y/n]? ')
+      sys.stdout.write('is this correct [y/N]? ')
       a = sys.stdin.readline().strip()
       if a in ('yes', 'y', 't', 'true'):
         break
@@ -230,7 +255,7 @@ to update the working directory files.
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
 
-    sys.stdout.write('Enable color display in this user account (y/n)? ')
+    sys.stdout.write('Enable color display in this user account (y/N)? ')
     a = sys.stdin.readline().strip().lower()
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
@@ -260,7 +285,8 @@ to update the working directory files.
     self._LinkManifest(opt.manifest_name)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-      self._ConfigureUser()
+      if opt.config_name or self._ShouldConfigureUser():
+        self._ConfigureUser()
       self._ConfigureColor()
 
     self._ConfigureDepth(opt)

subcmds/sync.py

@@ -131,12 +131,14 @@ later is required to fix a server side protocol bug.
     p.add_option('-d','--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
+    p.add_option('-c','--current-branch',
+                 dest='current_branch_only', action='store_true',
+                 help='fetch only current branch from server')
     p.add_option('-q','--quiet',
                  dest='quiet', action='store_true',
                  help='be more quiet')
     p.add_option('-j','--jobs',
                  dest='jobs', action='store', type='int',
-                 default=self.jobs,
                  help="projects to fetch simultaneously (default %d)" % self.jobs)
     if show_smart:
       p.add_option('-s', '--smart-sync',
@@ -180,7 +182,8 @@ later is required to fix a server side protocol bug.
       # - We always make sure we unlock the lock if we locked it.
       try:
         try:
-          success = project.Sync_NetworkHalf(quiet=opt.quiet)
+          success = project.Sync_NetworkHalf(quiet=opt.quiet,
+                                             current_branch_only=opt.current_branch_only)
 
           # Lock around all the rest of the code, since printing, updating a set
           # and Progress.update() are not thread safe.
@@ -196,15 +199,11 @@ later is required to fix a server side protocol bug.
 
           fetched.add(project.gitdir)
           pm.update()
-        except BaseException, e:
-          # Notify the _Fetch() function about all errors.
+        except _FetchError:
           err_event.set()
-
-          # If we got our own _FetchError, we don't want a stack trace.
-          # However, if we got something else (something in Sync_NetworkHalf?),
-          # we'd like one (so re-raise after we've set err_event).
-          if not isinstance(e, _FetchError):
-            raise
+        except:
+          err_event.set()
+          raise
       finally:
         if did_lock:
           lock.release()
@@ -217,7 +216,8 @@ later is required to fix a server side protocol bug.
     if self.jobs == 1:
       for project in projects:
         pm.update()
-        if project.Sync_NetworkHalf(quiet=opt.quiet):
+        if project.Sync_NetworkHalf(quiet=opt.quiet,
+                                    current_branch_only=opt.current_branch_only):
           fetched.add(project.gitdir)
         else:
           print >>sys.stderr, 'error: Cannot fetch %s' % project.name
@@ -393,7 +393,8 @@ uncommitted changes are present' % project.relpath
       _PostRepoUpgrade(self.manifest)
 
     if not opt.local_only:
-      mp.Sync_NetworkHalf(quiet=opt.quiet)
+      mp.Sync_NetworkHalf(quiet=opt.quiet,
+                          current_branch_only=opt.current_branch_only)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
@@ -401,6 +402,8 @@ uncommitted changes are present' % project.relpath
       if not syncbuf.Finish():
         sys.exit(1)
       self.manifest._Unload()
+      if opt.jobs is None:
+        self.jobs = self.manifest.default.sync_j
     all = self.GetProjects(args, missing_ok=True)
 
     if not opt.local_only:

subcmds/upload.py

@@ -73,7 +73,7 @@ Configuration
 
 review.URL.autoupload:
 
-To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+To disable the "Upload ... (y/N)?" prompt, you can set a per-project
 or global Git configuration option.  If review.URL.autoupload is set
 to "true" then repo will assume you always answer "y" at the prompt,
 and will not prompt you further.  If it is set to "false" then repo
@@ -162,7 +162,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       date = branch.date
       list = branch.commits
 
-      print 'Upload project %s/:' % project.relpath
+      print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
       print '  branch %s (%2d commit%s, %s):' % (
                     name,
                     len(list),
@@ -171,7 +171,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       for commit in list:
         print '         %s' % commit
 
-      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      sys.stdout.write('to %s (y/N)? ' % remote.review)
       answer = sys.stdin.readline().strip()
       answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
 
@@ -202,11 +202,12 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
         if b:
           script.append('#')
-        script.append('#  branch %s (%2d commit%s, %s):' % (
+        script.append('#  branch %s (%2d commit%s, %s) to remote branch %s:' % (
                       name,
                       len(list),
                       len(list) != 1 and 's' or '',
-                      date))
+                      date,
+                      project.revisionExpr))
         for commit in list:
           script.append('#         %s' % commit)
         b[name] = branch
@@ -215,6 +216,11 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       branches[project.name] = b
     script.append('')
 
+    script = [ x.encode('utf-8')
+             if issubclass(type(x), unicode)
+             else x
+             for x in script ]
+
     script = Editor.EditString("\n".join(script)).split("\n")
 
     project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
@@ -294,7 +300,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
 
             # if they want to auto upload, let's not ask because it could be automated
             if answer is None:
-                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ')
+                sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
                 a = sys.stdin.readline().strip().lower()
                 if a not in ('y', 'yes', 't', 'true', 'on'):
                     print >>sys.stderr, "skipping upload"

subcmds/version.py

@@ -19,6 +19,9 @@ from git_command import git
 from project import HEAD
 
 class Version(Command, MirrorSafeCommand):
+  wrapper_version = None
+  wrapper_path = None
+
   common = False
   helpSummary = "Display the version of repo"
   helpUsage = """
@@ -31,5 +34,10 @@ class Version(Command, MirrorSafeCommand):
 
     print 'repo version %s' % rp.work_git.describe(HEAD)
     print '       (from %s)' % rem.url
+
+    if Version.wrapper_path is not None:
+      print 'repo launcher version %s' % Version.wrapper_version
+      print '       (from %s)' % Version.wrapper_path
+
     print git.version().strip()
     print 'Python %s' % sys.version