Remove extra '/' in RemoteSpec

urljoin appends a '/' if only the domain is in the url path.  This
change strips that off before creating a RemoteSpec

git_config.py

@@ -26,7 +26,6 @@ import time
 import urllib2
 
 from signal import SIGTERM
-from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
 
@@ -576,9 +575,19 @@ class Remote(object):
         self._review_protocol = info[0]
         self._review_host = info[1]
         self._review_port = info[2]
+      elif 'REPO_HOST_PORT_INFO' in os.environ:
+        info = os.environ['REPO_HOST_PORT_INFO']
+        self._review_protocol = 'ssh'
+        self._review_host = info.split(" ")[0]
+        self._review_port = info.split(" ")[1]
+
+        REVIEW_CACHE[u] = (
+          self._review_protocol,
+          self._review_host,
+          self._review_port)
       else:
         try:
-          info = urlopen(u).read()
+          info = urllib2.urlopen(u).read()
           if info == 'NOT_AVAILABLE':
             raise UploadError('%s: SSH disabled' % self.review)
           if '<' in info:
@@ -590,15 +599,15 @@ class Remote(object):
           self._review_protocol = 'ssh'
           self._review_host = info.split(" ")[0]
           self._review_port = info.split(" ")[1]
-        except urllib2.URLError, e:
-          raise UploadError('%s: %s' % (self.review, e.reason[1]))
-        except HTTPError, e:
+        except urllib2.HTTPError, e:
           if e.code == 404:
             self._review_protocol = 'http-post'
             self._review_host = None
             self._review_port = None
           else:
-            raise UploadError('Upload over ssh unavailable')
+            raise UploadError('Upload over SSH unavailable')
+        except urllib2.URLError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
 
         REVIEW_CACHE[u] = (
           self._review_protocol,

main.py

@@ -273,6 +273,24 @@ class _UserAgentHandler(urllib2.BaseHandler):
     req.add_header('User-Agent', _UserAgent())
     return req
 
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_auth_reqed(self, authreq, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+        self, authreq, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -287,7 +305,7 @@ def init_http():
     pass
   except IOError:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(_BasicAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']

manifest_xml.py

@@ -46,16 +46,20 @@ class _XmlRemote(object):
     self.fetchUrl = fetch
     self.manifestUrl = manifestUrl
     self.reviewUrl = review
+    self.resolvedFetchUrl = self._resolveFetchUrl()
 
-  def ToRemoteSpec(self, projectName):
-    url = self.fetchUrl.rstrip('/') + '/' + projectName + '.git'
+  def _resolveFetchUrl(self):
+    url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
     # urljoin will get confused if there is no scheme in the base url
     # ie, if manifestUrl is of the form <hostname:port>
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
         manifestUrl = 'gopher://' + manifestUrl
     url = urlparse.urljoin(manifestUrl, url)
-    url = re.sub(r'^gopher://', '', url)
+    return re.sub(r'^gopher://', '', url)
+
+  def ToRemoteSpec(self, projectName):
+    url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
     return RemoteSpec(self.name, url, self.reviewUrl)
 
 class XmlManifest(object):
@@ -368,7 +372,7 @@ class XmlManifest(object):
       raise ManifestParseError, 'refusing to mirror %s' % m_url
 
     if self._default and self._default.remote:
-      url = self._default.remote.fetchUrl
+      url = self._default.remote.resolvedFetchUrl
       if not url.endswith('/'):
         url += '/'
       if m_url.startswith(url):

project.py

@@ -16,12 +16,24 @@ import traceback
 import errno
 import filecmp
 import os
+import random
 import re
 import shutil
 import stat
 import sys
+import time
 import urllib2
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
+try:
+  from os import SEEK_END
+except ImportError:
+  SEEK_END = 2
+
 from color import Coloring
 from git_command import GitCommand
 from git_config import GitConfig, IsId, GetSchemeFromUrl
@@ -32,6 +44,8 @@ from progress import Progress
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
+_urllib_lock = _threading.Lock()
+
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -894,9 +908,25 @@ class Project(object):
       is_new = not self.Exists
     if is_new:
       self._InitGitDir()
-
     self._InitRemote()
-    if not self._RemoteFetch(initial=is_new, quiet=quiet):
+
+    if is_new:
+      alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      try:
+        fd = open(alt, 'rb')
+        try:
+          alt_dir = fd.readline().rstrip()
+        finally:
+          fd.close()
+      except IOError:
+        alt_dir = None
+    else:
+      alt_dir = None
+
+    if alt_dir is None and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+      is_new = False
+
+    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir):
       return False
 
     #Check that the requested ref was found after fetch
@@ -1307,7 +1337,8 @@ class Project(object):
 
   def _RemoteFetch(self, name=None, tag=None,
                    initial=False,
-                   quiet=False):
+                   quiet=False,
+                   alt_dir=None):
     if not name:
       name = self.remote.name
 
@@ -1316,29 +1347,9 @@ class Project(object):
     if remote.PreConnectFetch():
       ssh_proxy = True
 
-    bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
-    bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
-    use_bundle = False
-    if os.path.exists(bundle_dst) or os.path.exists(bundle_tmp):
-      use_bundle = True
-
     if initial:
-      alt = os.path.join(self.gitdir, 'objects/info/alternates')
-      try:
-        fd = open(alt, 'rb')
-        try:
-          ref_dir = fd.readline()
-          if ref_dir and ref_dir.endswith('\n'):
-            ref_dir = ref_dir[:-1]
-        finally:
-          fd.close()
-      except IOError, e:
-        ref_dir = None
-
-      if ref_dir and 'objects' == os.path.basename(ref_dir):
-        if use_bundle:
-          use_bundle = False
-        ref_dir = os.path.dirname(ref_dir)
+      if alt_dir and 'objects' == os.path.basename(alt_dir):
+        ref_dir = os.path.dirname(alt_dir)
         packed_refs = os.path.join(self.gitdir, 'packed-refs')
         remote = self.GetRemote(name)
 
@@ -1374,10 +1385,8 @@ class Project(object):
             old_packed += line
 
         _lwrite(packed_refs, tmp_packed)
-
       else:
-        ref_dir = None
-        use_bundle = True
+        alt_dir = None
 
     cmd = ['fetch']
 
@@ -1386,94 +1395,119 @@ class Project(object):
     depth = self.manifest.manifestProject.config.GetString('repo.depth')
     if depth and initial:
       cmd.append('--depth=%s' % depth)
-      use_bundle = False
 
     if quiet:
       cmd.append('--quiet')
     if not self.worktree:
       cmd.append('--update-head-ok')
+    cmd.append(name)
+    if tag is not None:
+      cmd.append('tag')
+      cmd.append(tag)
 
-    if use_bundle and not os.path.exists(bundle_dst):
-      bundle_url = remote.url + '/clone.bundle'
-      bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
-      if GetSchemeFromUrl(bundle_url) in ('http', 'https'):
-        use_bundle = self._FetchBundle(
-          bundle_url,
-          bundle_tmp,
-          bundle_dst,
-          quiet=quiet)
-      else:
-        use_bundle = False
-
-    if use_bundle:
-      if not quiet:
-        cmd.append('--quiet')
-      cmd.append(bundle_dst)
-      for f in remote.fetch:
-        cmd.append(str(f))
-      cmd.append('refs/tags/*:refs/tags/*')
-    else:
-      cmd.append(name)
-      if tag is not None:
-        cmd.append('tag')
-        cmd.append(tag)
-
-    ok = GitCommand(self,
-                    cmd,
-                    bare = True,
-                    ssh_proxy = ssh_proxy).Wait() == 0
+    ok = False
+    for i in range(2):
+      if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0:
+        ok = True
+        break
+      time.sleep(random.randint(30, 45))
 
     if initial:
-      if ref_dir:
+      if alt_dir:
         if old_packed != '':
           _lwrite(packed_refs, old_packed)
         else:
           os.remove(packed_refs)
       self.bare_git.pack_refs('--all', '--prune')
+    return ok
 
+  def _ApplyCloneBundle(self, initial=False, quiet=False):
+    if initial and self.manifest.manifestProject.config.GetString('repo.depth'):
+      return False
+
+    remote = self.GetRemote(self.remote.name)
+    bundle_url = remote.url + '/clone.bundle'
+    bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+    if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
+      return False
+
+    bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
+    bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
+
+    exist_dst = os.path.exists(bundle_dst)
+    exist_tmp = os.path.exists(bundle_tmp)
+
+    if not initial and not exist_dst and not exist_tmp:
+      return False
+
+    if not exist_dst:
+      exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
+    if not exist_dst:
+      return False
+
+    cmd = ['fetch']
+    if quiet:
+      cmd.append('--quiet')
+    if not self.worktree:
+      cmd.append('--update-head-ok')
+    cmd.append(bundle_dst)
+    for f in remote.fetch:
+      cmd.append(str(f))
+    cmd.append('refs/tags/*:refs/tags/*')
+
+    ok = GitCommand(self, cmd, bare=True).Wait() == 0
     if os.path.exists(bundle_dst):
       os.remove(bundle_dst)
     if os.path.exists(bundle_tmp):
       os.remove(bundle_tmp)
-
     return ok
 
-  def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet=False):
+  def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
     keep = True
     done = False
     dest = open(tmpPath, 'a+b')
     try:
-      dest.seek(0, os.SEEK_END)
+      dest.seek(0, SEEK_END)
       pos = dest.tell()
 
-      req = urllib2.Request(srcUrl)
-      if pos > 0:
-        req.add_header('Range', 'bytes=%d-' % pos)
-
+      _urllib_lock.acquire()
       try:
-        r = urllib2.urlopen(req)
-      except urllib2.HTTPError, e:
-        if e.code == 404:
-          keep = False
-          return False
-        elif e.info()['content-type'] == 'text/plain':
-          try:
-            msg = e.read()
-            if len(msg) > 0 and msg[-1] == '\n':
-              msg = msg[0:-1]
-            msg = ' (%s)' % msg
-          except:
-            msg = ''
-        else:
-          try:
-            from BaseHTTPServer import BaseHTTPRequestHandler
-            res = BaseHTTPRequestHandler.responses[e.code]
-            msg = ' (%s: %s)' % (res[0], res[1])
-          except:
-            msg = ''
-        raise DownloadError('HTTP %s%s' % (e.code, msg))
-      except urllib2.URLError, e:
-        raise DownloadError('%s (%s)' % (e.reason, req.get_host()))
+        req = urllib2.Request(srcUrl)
+        if pos > 0:
+          req.add_header('Range', 'bytes=%d-' % pos)
+
+        try:
+          r = urllib2.urlopen(req)
+        except urllib2.HTTPError, e:
+          def _content_type():
+            try:
+              return e.info()['content-type']
+            except:
+              return None
+
+          if e.code == 404:
+            keep = False
+            return False
+          elif _content_type() == 'text/plain':
+            try:
+              msg = e.read()
+              if len(msg) > 0 and msg[-1] == '\n':
+                msg = msg[0:-1]
+              msg = ' (%s)' % msg
+            except:
+              msg = ''
+          else:
+            try:
+              from BaseHTTPServer import BaseHTTPRequestHandler
+              res = BaseHTTPRequestHandler.responses[e.code]
+              msg = ' (%s: %s)' % (res[0], res[1])
+            except:
+              msg = ''
+          raise DownloadError('HTTP %s%s' % (e.code, msg))
+        except urllib2.URLError, e:
+          raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+      finally:
+        _urllib_lock.release()
 
       p = None
       try:

subcmds/help.py

@@ -165,6 +165,7 @@ See 'repo help --all' for a complete list of recognized commands.
         print >>sys.stderr, "repo: '%s' is not a repo command." % name
         sys.exit(1)
 
+      cmd.manifest = self.manifest
       self._PrintCommandHelp(cmd)
 
     else:

subcmds/sync.py

@@ -195,15 +195,11 @@ later is required to fix a server side protocol bug.
 
           fetched.add(project.gitdir)
           pm.update()
-        except BaseException, e:
-          # Notify the _Fetch() function about all errors.
+        except _FetchError:
           err_event.set()
-
-          # If we got our own _FetchError, we don't want a stack trace.
-          # However, if we got something else (something in Sync_NetworkHalf?),
-          # we'd like one (so re-raise after we've set err_event).
-          if not isinstance(e, _FetchError):
-            raise
+        except:
+          err_event.set()
+          raise
       finally:
         if did_lock:
           lock.release()