Ignore /clone.bundle on HTTP 401, 403 and 404

401: Unauthorized, authentication may be required. This is usually
     handled internally by the HTTP client in Python. If it reaches
     our code in repo, the Python HTTP client didn't find a password
     in ~/.netrc that it could use.

403: Authentication was supplied, but is incorrect. It might be
     that the CDN doesn't want to offer this clone.bundle file
     to the client, but the Git fetch operation would still be
     successful. This might arise if branch level read controls
     were used in Gerrit Code Review and the /clone.bundle file
     contained branches not visible to the client.

404: The server has no /clone.bundle file available.

In all of these cases, sliently ignore the /clone.bundle file HTTP
error and let the Git operation take over.

Change-Id: I1787f3cac86b017351952bbb793fe5874d83c72b

SUBMITTING_PATCHES

@@ -5,7 +5,7 @@ Short Version:
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review:
 
-   git push ssh://review.source.android.com:29418/tools/repo.git HEAD:refs/for/master
+   git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/maint
 
 
 Long Version:
@@ -55,24 +55,23 @@ Do not email your patches to anyone.
 
 Instead, login to the Gerrit Code Review tool at:
 
-  https://review.source.android.com/
+  https://gerrit-review.googlesource.com/
 
 Ensure you have completed one of the necessary contributor
 agreements, providing documentation to the project maintainers that
 they have right to redistribute your work under the Apache License:
 
-  https://review.source.android.com/#settings,agreements
+  https://gerrit-review.googlesource.com/#/settings/agreements
 
-Ensure you have registered one or more SSH public keys, so you can
+Ensure you have obtained an HTTP password to authenticate:
-push your commits directly over SSH:
 
-  https://review.source.android.com/#settings,ssh-keys
+  https://gerrit-review.googlesource.com/new-password
 
-Push your patches over SSH to the review server, possibly through
+Push your patches over HTTPS to the review server, possibly through
 a remembered remote to make this easier in the future:
 
-   git config remote.review.url ssh://review.source.android.com:29418/tools/repo.git
+   git config remote.review.url https://gerrit-review.googlesource.com/git-repo
-   git config remote.review.push HEAD:refs/for/master
+   git config remote.review.push HEAD:refs/for/maint
 
    git push review
 

git_config.py

@@ -488,7 +488,7 @@ def close_ssh():
   _master_keys_lock = None
 
 URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
-URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
+URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
 
 def GetSchemeFromUrl(url):
   m = URI_ALL.match(url)
@@ -527,7 +527,7 @@ class Remote(object):
     self.projectname = self._Get('projectname')
     self.fetch = map(lambda x: RefSpec.FromString(x),
                      self._Get('fetch', all=True))
-    self._review_protocol = None
+    self._review_url = None
 
   def _InsteadOf(self):
     globCfg = GitConfig.ForUser()
@@ -554,9 +554,8 @@ class Remote(object):
     connectionUrl = self._InsteadOf()
     return _preconnect(connectionUrl)
 
-  @property
+  def ReviewUrl(self, userEmail):
-  def ReviewProtocol(self):
+    if self._review_url is None:
-    if self._review_protocol is None:
       if self.review is None:
         return None
 
@@ -565,67 +564,47 @@ class Remote(object):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
         u = u[:len(u) - len('/Gerrit')]
-      if not u.endswith('/ssh_info'):
+      if u.endswith('/ssh_info'):
+        u = u[:len(u) - len('/ssh_info')]
       if not u.endswith('/'):
          u += '/'
-        u += 'ssh_info'
+      http_url = u
 
       if u in REVIEW_CACHE:
-        info = REVIEW_CACHE[u]
+        self._review_url = REVIEW_CACHE[u]
-        self._review_protocol = info[0]
-        self._review_host = info[1]
-        self._review_port = info[2]
       elif 'REPO_HOST_PORT_INFO' in os.environ:
-        info = os.environ['REPO_HOST_PORT_INFO']
+        host, port = os.environ['REPO_HOST_PORT_INFO'].split()
-        self._review_protocol = 'ssh'
+        self._review_url = self._SshReviewUrl(userEmail, host, port)
-        self._review_host = info.split(" ")[0]
+        REVIEW_CACHE[u] = self._review_url
-        self._review_port = info.split(" ")[1]
-
-        REVIEW_CACHE[u] = (
-          self._review_protocol,
-          self._review_host,
-          self._review_port)
       else:
         try:
-          info = urllib2.urlopen(u).read()
+          info_url = u + 'ssh_info'
-          if info == 'NOT_AVAILABLE':
+          info = urllib2.urlopen(info_url).read()
-            raise UploadError('%s: SSH disabled' % self.review)
           if '<' in info:
             # Assume the server gave us some sort of HTML
             # response back, like maybe a login page.
             #
-            raise UploadError('%s: Cannot parse response' % u)
+            raise UploadError('%s: Cannot parse response' % info_url)
 
-          self._review_protocol = 'ssh'
+          if info == 'NOT_AVAILABLE':
-          self._review_host = info.split(" ")[0]
+            # Assume HTTP if SSH is not enabled.
-          self._review_port = info.split(" ")[1]
+            self._review_url = http_url + 'p/'
-        except urllib2.HTTPError, e:
-          if e.code == 404:
-            self._review_protocol = 'http-post'
-            self._review_host = None
-            self._review_port = None
           else:
-            raise UploadError('Upload over SSH unavailable')
+            host, port = info.split()
+            self._review_url = self._SshReviewUrl(userEmail, host, port)
+        except urllib2.HTTPError, e:
+          raise UploadError('%s: %s' % (self.review, str(e)))
         except urllib2.URLError, e:
           raise UploadError('%s: %s' % (self.review, str(e)))
 
-        REVIEW_CACHE[u] = (
+        REVIEW_CACHE[u] = self._review_url
-          self._review_protocol,
+    return self._review_url + self.projectname
-          self._review_host,
-          self._review_port)
-    return self._review_protocol
 
-  def SshReviewUrl(self, userEmail):
+  def _SshReviewUrl(self, userEmail, host, port):
-    if self.ReviewProtocol != 'ssh':
-      return None
     username = self._config.GetString('review.%s.username' % self.review)
     if username is None:
-      username = userEmail.split("@")[0]
+      username = userEmail.split('@')[0]
-    return 'ssh://%s@%s:%s/%s' % (
+    return 'ssh://%s@%s:%s/' % (username, host, port)
-      username,
-      self._review_host,
-      self._review_port,
-      self.projectname)
 
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.

main.py

@@ -36,6 +36,7 @@ from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
+from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
 from error import ManifestInvalidRevisionError
@@ -294,6 +295,24 @@ class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
         self.retried = 0
       raise
 
+class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+  def http_error_auth_reqed(self, auth_header, host, req, headers):
+    try:
+      old_add_header = req.add_header
+      def _add_header(name, val):
+        val = val.replace('\n', '')
+        old_add_header(name, val)
+      req.add_header = _add_header
+      return urllib2.AbstractDigestAuthHandler.http_error_auth_reqed(
+        self, auth_header, host, req, headers)
+    except:
+      reset = getattr(self, 'reset_retry_count', None)
+      if reset is not None:
+        reset()
+      elif getattr(self, 'retried', None):
+        self.retried = 0
+      raise
+
 def init_http():
   handlers = [_UserAgentHandler()]
 
@@ -302,13 +321,14 @@ def init_http():
     n = netrc.netrc()
     for host in n.hosts:
       p = n.hosts[host]
-      mgr.add_password(None, 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(p[1], 'http://%s/'  % host, p[0], p[2])
-      mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
+      mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
   except netrc.NetrcParseError:
     pass
   except IOError:
     pass
   handlers.append(_BasicAuthHandler(mgr))
+  handlers.append(_DigestAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']
@@ -334,6 +354,9 @@ def _Main(argv):
   _CheckWrapperVersion(opt.wrapper_version, opt.wrapper_path)
   _CheckRepoDir(opt.repodir)
 
+  Version.wrapper_version = opt.wrapper_version
+  Version.wrapper_path = opt.wrapper_path
+
   repo = _Repo(opt.repodir)
   try:
     try:

manifest_xml.py

@@ -228,12 +228,8 @@ class XmlManifest(object):
   @property
   def manifest_server(self):
     self._Load()
-
-    if self._manifest_server:
     return self._manifest_server
 
-    return self.manifestProject.config.GetString('repo.manifest-server')
-
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
@@ -502,6 +498,12 @@ class XmlManifest(object):
             "project %s path cannot be absolute in %s" % \
             (name, self.manifestFile)
 
+    rebase = node.getAttribute('rebase')
+    if not rebase:
+      rebase = True
+    else:
+      rebase = rebase.lower() in ("yes", "true", "1")
+
     if self.IsMirror:
       relpath = None
       worktree = None
@@ -517,7 +519,8 @@ class XmlManifest(object):
                       worktree = worktree,
                       relpath = path,
                       revisionExpr = revisionExpr,
-                      revisionId = None)
+                      revisionId = None,
+                      rebase = rebase)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':

project.py

@@ -503,7 +503,8 @@ class Project(object):
                worktree,
                relpath,
                revisionExpr,
-               revisionId):
+               revisionId,
+               rebase = True):
     self.manifest = manifest
     self.name = name
     self.remote = remote
@@ -522,6 +523,8 @@ class Project(object):
     else:
       self.revisionId = revisionId
 
+    self.rebase = rebase
+
     self.snapshots = {}
     self.copyfiles = []
     self.config = GitConfig.ForRepository(
@@ -688,7 +691,7 @@ class Project(object):
     di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
-    if not rb and not di and not df and not do:
+    if not rb and not di and not df and not do and not self.CurrentBranch:
       return 'CLEAN'
 
     out = StatusColoring(self.config)
@@ -866,32 +869,31 @@ class Project(object):
       branch.remote.projectname = self.name
       branch.remote.Save()
 
-    if branch.remote.ReviewProtocol == 'ssh':
+    url = branch.remote.ReviewUrl(self.UserEmail)
-      if dest_branch.startswith(R_HEADS):
+    if url is None:
-        dest_branch = dest_branch[len(R_HEADS):]
+      raise UploadError('review not configured')
+    cmd = ['push']
 
+    if url.startswith('ssh://'):
       rp = ['gerrit receive-pack']
       for e in people[0]:
         rp.append('--reviewer=%s' % sq(e))
       for e in people[1]:
         rp.append('--cc=%s' % sq(e))
+      cmd.append('--receive-pack=%s' % " ".join(rp))
 
+    cmd.append(url)
+
+    if dest_branch.startswith(R_HEADS):
+      dest_branch = dest_branch[len(R_HEADS):]
     ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
     if auto_topic:
       ref_spec = ref_spec + '/' + branch.name
-
-      cmd = ['push']
-      cmd.append('--receive-pack=%s' % " ".join(rp))
-      cmd.append(branch.remote.SshReviewUrl(self.UserEmail))
     cmd.append(ref_spec)
 
     if GitCommand(self, cmd, bare = True).Wait() != 0:
       raise UploadError('Upload failed')
 
-    else:
-        raise UploadError('Unsupported protocol %s' \
-          % branch.remote.review)
-
     msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
     self.bare_git.UpdateRef(R_PUB + branch.name,
                             R_HEADS + branch.name,
@@ -900,7 +902,11 @@ class Project(object):
 
 ## Sync ##
 
-  def Sync_NetworkHalf(self, quiet=False, is_new=None, current_branch_only=False):
+  def Sync_NetworkHalf(self,
+      quiet=False,
+      is_new=None,
+      current_branch_only=False,
+      clone_bundle=True):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
@@ -923,7 +929,9 @@ class Project(object):
     else:
       alt_dir = None
 
-    if alt_dir is None and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+    if clone_bundle \
+    and alt_dir is None \
+    and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
       is_new = False
 
     if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
@@ -1094,10 +1102,12 @@ class Project(object):
                    len(local_changes) - cnt_mine)
 
     branch.remote = self.GetRemote(self.remote.name)
+    if not ID_RE.match(self.revisionExpr):
+      # in case of manifest sync the revisionExpr might be a SHA1
       branch.merge = self.revisionExpr
     branch.Save()
 
-    if cnt_mine > 0:
+    if cnt_mine > 0 and self.rebase:
       def _dorebase():
         self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
         self._CopyFiles()
@@ -1447,6 +1457,8 @@ class Project(object):
     remote = self.GetRemote(self.remote.name)
     bundle_url = remote.url + '/clone.bundle'
     bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+    if GetSchemeFromUrl(bundle_url) in ('persistent-http', 'persistent-https'):
+      bundle_url = bundle_url[len('persistent-'):]
     if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
       return False
 
@@ -1504,7 +1516,7 @@ class Project(object):
             except:
               return None
 
-          if e.code == 404:
+          if e.code in (401, 403, 404):
             keep = False
             return False
           elif _content_type() == 'text/plain':
@@ -1530,7 +1542,7 @@ class Project(object):
 
       p = None
       try:
-        size = r.headers['content-length']
+        size = r.headers.get('content-length', 0)
         unit = 1 << 10
 
         if size and not quiet:

repo

@@ -311,11 +311,12 @@ def _InitHttp():
     n = netrc.netrc()
     for host in n.hosts:
       p = n.hosts[host]
-      mgr.add_password(None, 'http://%s/'  % host, p[0], p[2])
+      mgr.add_password(p[1], 'http://%s/'  % host, p[0], p[2])
-      mgr.add_password(None, 'https://%s/' % host, p[0], p[2])
+      mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
   except:
     pass
   handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+  handlers.append(urllib2.HTTPDigestAuthHandler(mgr))
 
   if 'http_proxy' in os.environ:
     url = os.environ['http_proxy']

subcmds/init.py

@@ -23,7 +23,6 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION
-from git_config import GitConfig
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
@@ -37,20 +36,6 @@ The latest repo source code and manifest collection is downloaded
 from the server and is installed in the .repo/ directory in the
 current working directory.
 
-The optional -u argument can be used to specify a URL to the
-manifest project. It is also possible to have a git configuration
-section as below to use 'identifier' as argument to -u:
-
-  [repo-manifest "identifier"]
-    url = ...
-    server = ...
-    reference = ...
-
-Only the url is required - the others are optional.
-
-If no -u argument is specified, the 'repo-manifest' section named
-'default' will be used if present.
-
 The optional -b argument can be used to select the manifest branch
 to checkout and use.  If no branch is specified, master is assumed.
 
@@ -84,7 +69,7 @@ to update the working directory files.
     # Manifest
     g = p.add_option_group('Manifest options')
     g.add_option('-u', '--manifest-url',
-                 dest='manifest_url', default='default',
+                 dest='manifest_url',
                  help='manifest repository location', metavar='URL')
     g.add_option('-b', '--manifest-branch',
                  dest='manifest_branch',
@@ -123,25 +108,10 @@ to update the working directory files.
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
-    manifest_server = None
-
-    # The manifest url may point out a manifest section in the config
-    key = 'repo-manifest.%s.' % opt.manifest_url
-    if GitConfig.ForUser().GetString(key + 'url'):
-      opt.manifest_url = GitConfig.ForUser().GetString(key + 'url')
-
-      # Also copy other options to the manifest config if not specified already.
-      if not opt.reference:
-        opt.reference = GitConfig.ForUser().GetString(key + 'reference')
-      manifest_server = GitConfig.ForUser().GetString(key + 'server')
 
     if is_new:
-      if not opt.manifest_url or opt.manifest_url == 'default':
+      if not opt.manifest_url:
-        print >>sys.stderr, """\
+        print >>sys.stderr, 'fatal: manifest url (-u) is required.'
-fatal: missing manifest url (-u) and no default found.
-
-  tip: The global git configuration key 'repo-manifest.default.url' can
-       be used to specify a default url."""
         sys.exit(1)
 
       if not opt.quiet:
@@ -165,9 +135,6 @@ fatal: missing manifest url (-u) and no default found.
       r.ResetFetch()
       r.Save()
 
-    if manifest_server:
-      m.config.SetString('repo.manifest-server', manifest_server)
-
     if opt.reference:
       m.config.SetString('repo.reference', opt.reference)
 

subcmds/status.py

@@ -101,19 +101,6 @@ the following meanings:
     all = self.GetProjects(args)
     counter = itertools.count()
 
-    on = {}
-    for project in all:
-      cb = project.CurrentBranch
-      if cb:
-        if cb not in on:
-          on[cb] = []
-        on[cb].append(project)
-
-    branch_names = list(on.keys())
-    branch_names.sort()
-    for cb in branch_names:
-      print '# on branch %s' % cb
-
     if opt.jobs == 1:
       for project in all:
         state = project.PrintWorkTreeStatus()

subcmds/sync.py

@@ -86,6 +86,12 @@ specify a custom tag/label.
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
 
+The --no-clone-bundle option disables any attempt to use
+$URL/clone.bundle to bootstrap a new Git repository from a
+resumeable bundle file on a content delivery network. This
+may be necessary if there are problems with the local Python
+HTTP client or proxy configuration, but the Git binary works.
+
 SSH Connections
 ---------------
 
@@ -140,6 +146,12 @@ later is required to fix a server side protocol bug.
     p.add_option('-j','--jobs',
                  dest='jobs', action='store', type='int',
                  help="projects to fetch simultaneously (default %d)" % self.jobs)
+    p.add_option('-m', '--manifest-name',
+                 dest='manifest_name',
+                 help='temporary manifest to use for this sync', metavar='NAME.xml')
+    p.add_option('--no-clone-bundle',
+                 dest='no_clone_bundle', action='store_true',
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -182,8 +194,10 @@ later is required to fix a server side protocol bug.
       # - We always make sure we unlock the lock if we locked it.
       try:
         try:
-          success = project.Sync_NetworkHalf(quiet=opt.quiet,
+          success = project.Sync_NetworkHalf(
-                                             current_branch_only=opt.current_branch_only)
+            quiet=opt.quiet,
+            current_branch_only=opt.current_branch_only,
+            clone_bundle=not opt.no_clone_bundle)
 
           # Lock around all the rest of the code, since printing, updating a set
           # and Progress.update() are not thread safe.
@@ -333,6 +347,15 @@ uncommitted changes are present' % project.relpath
     if opt.network_only and opt.local_only:
       print >>sys.stderr, 'error: cannot combine -n and -l'
       sys.exit(1)
+    if opt.manifest_name and opt.smart_sync:
+      print >>sys.stderr, 'error: cannot combine -m and -s'
+      sys.exit(1)
+    if opt.manifest_name and opt.smart_tag:
+      print >>sys.stderr, 'error: cannot combine -m and -t'
+      sys.exit(1)
+
+    if opt.manifest_name:
+      self.manifest.Override(opt.manifest_name)
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:

subcmds/version.py

@@ -19,6 +19,9 @@ from git_command import git
 from project import HEAD
 
 class Version(Command, MirrorSafeCommand):
+  wrapper_version = None
+  wrapper_path = None
+
   common = False
   helpSummary = "Display the version of repo"
   helpUsage = """
@@ -31,5 +34,10 @@ class Version(Command, MirrorSafeCommand):
 
     print 'repo version %s' % rp.work_git.describe(HEAD)
     print '       (from %s)' % rem.url
+
+    if Version.wrapper_path is not None:
+      print 'repo launcher version %s' % Version.wrapper_version
+      print '       (from %s)' % Version.wrapper_path
+
     print git.version().strip()
     print 'Python %s' % sys.version