Work around Python 2.7 urllib2 bug

If the remote is using authenticated HTTP, but does not have
$GIT_URL/clone.bundle files in each repository, an initial sync
would fail around 8 projects in due to the library not resetting
the number of failures after getting a 404.

Work around this by updating the retry counter ourselves.

The urllib2 library is also not thread-safe. Make it somewhat
safer by wrapping the critical section with a lock.

Change-Id: I886e2750ef4793cbe2150c3b5396eb9f10974f7f
Signed-off-by: Shawn O. Pearce <sop@google.com>

main.py

@@ -273,6 +273,15 @@ class _UserAgentHandler(urllib2.BaseHandler):
     req.add_header('User-Agent', _UserAgent())
     return req
 
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+  def http_error_auth_reqed(self, authreq, host, req, headers):
+    try:
+      return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+        self, authreq, host, req, headers)
+    except:
+      self.reset_retry_count()
+      raise
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -287,7 +296,7 @@ def init_http():
     pass
   except IOError:
     pass
-  handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(_BasicAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']

project.py

@@ -24,6 +24,11 @@ import sys
 import time
 import urllib2
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
 from color import Coloring
 from git_command import GitCommand
 from git_config import GitConfig, IsId, GetSchemeFromUrl
@@ -34,6 +39,8 @@ from progress import Progress
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
+_urllib_lock = _threading.Lock()
+
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -1458,6 +1465,8 @@ class Project(object):
       dest.seek(0, os.SEEK_END)
       pos = dest.tell()
 
+      _urllib_lock.acquire()
+      try:
         req = urllib2.Request(srcUrl)
         if pos > 0:
           req.add_header('Range', 'bytes=%d-' % pos)
@@ -1492,6 +1501,8 @@ class Project(object):
           raise DownloadError('HTTP %s%s' % (e.code, msg))
         except urllib2.URLError, e:
           raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+      finally:
+        _urllib_lock.release()
 
       p = None
       try: