Use curl command line tool for clone.bundle

urllib2 is not thread safe and may be causing sync to lock up or
not work correctly on various platforms. Instead use the command
line curl program.

Change-Id: I36eaf18bb4df089d26ea99d533cb015e7c616eb0

docs/manifest-format.txt

@@ -32,6 +32,7 @@ following DTD:
   
     <!ELEMENT remote (EMPTY)>
     <!ATTLIST remote name         ID    #REQUIRED>
+    <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
   
@@ -89,6 +90,12 @@ name specified here is used as the remote name in each project's
 .git/config, and is therefore automatically available to commands
 like `git fetch`, `git remote`, `git pull` and `git push`.
 
+Attribute `alias`: The alias, if specified, is used to override
+`name` to be set as the remote name in each project's .git/config.
+Its value can be duplicated while attribute `name` has to be unique
+in the manifest file. This helps each project to be able to have
+same remote name which actually points to different remote url.
+
 Attribute `fetch`: The Git URL prefix for all projects which use
 this remote.  Each project's name is appended to this prefix to
 form the actual URL used to clone the project.
@@ -171,7 +178,11 @@ the default element is used.
 
 Attribute `groups`: List of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
-"default".
+"default", and each project automatically belongs to a group of
+it's name:`name` and path:`path`.  E.g. for
+<project name="monkeys" path="barrel-of"/>, that project
+definition is implicitly in the following manifest groups:
+default, name:monkeys, and path:barrel-of.
 
 Element annotation
 ------------------

git_command.py

@@ -89,7 +89,7 @@ class _GitCall(object):
       if ver_str.startswith('git version '):
         _git_version = tuple(
           map(lambda x: int(x),
-            ver_str[len('git version '):].strip().split('.')[0:3]
+            ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
           ))
       else:
         print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
@@ -148,7 +148,7 @@ class GitCommand(object):
       _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
       _setenv(env, 'GIT_SSH', _ssh_proxy())
     if 'http_proxy' in env and 'darwin' == sys.platform:
-      s = 'http.proxy=' + env['http_proxy']
+      s = "'http.proxy=%s'" % (env['http_proxy'],)
       p = env.get('GIT_CONFIG_PARAMETERS')
       if p is not None:
         s = p + ' ' + s

manifest_xml.py

@@ -41,12 +41,14 @@ class _Default(object):
 class _XmlRemote(object):
   def __init__(self,
                name,
+               alias=None,
                fetch=None,
                manifestUrl=None,
                review=None):
     self.name = name
     self.fetchUrl = fetch
     self.manifestUrl = manifestUrl
+    self.remoteAlias = alias
     self.reviewUrl = review
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
@@ -62,7 +64,10 @@ class _XmlRemote(object):
 
   def ToRemoteSpec(self, projectName):
     url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
-    return RemoteSpec(self.name, url, self.reviewUrl)
+    remoteName = self.name
+    if self.remoteAlias:
+      remoteName = self.remoteAlias
+    return RemoteSpec(remoteName, url, self.reviewUrl)
 
 class XmlManifest(object):
   """manages the repo configuration file"""
@@ -283,11 +288,12 @@ class XmlManifest(object):
       self.branch = b
 
       nodes = []
-      nodes.append(self._ParseManifestXml(self.manifestFile))
+      nodes.append(self._ParseManifestXml(self.manifestFile,
+                                          self.manifestProject.worktree))
 
       local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
       if os.path.exists(local):
-        nodes.append(self._ParseManifestXml(local))
+        nodes.append(self._ParseManifestXml(local, self.repodir))
 
       self._ParseManifest(nodes)
 
@@ -297,7 +303,7 @@ class XmlManifest(object):
 
       self._loaded = True
 
-  def _ParseManifestXml(self, path):
+  def _ParseManifestXml(self, path, include_root):
     root = xml.dom.minidom.parse(path)
     if not root or not root.childNodes:
       raise ManifestParseError("no root node in %s" % (path,))
@@ -310,13 +316,13 @@ class XmlManifest(object):
     for node in config.childNodes:
         if node.nodeName == 'include':
             name = self._reqatt(node, 'name')
-            fp = os.path.join(os.path.dirname(path), name)
+            fp = os.path.join(include_root, name)
             if not os.path.isfile(fp):
                 raise ManifestParseError, \
                     "include %s doesn't exist or isn't a file" % \
                     (name,)
             try:
-                nodes.extend(self._ParseManifestXml(fp))
+                nodes.extend(self._ParseManifestXml(fp, include_root))
             # should isolate this to the exact exception, but that's
             # tricky.  actual parsing implementation may vary.
             except (KeyboardInterrupt, RuntimeError, SystemExit):
@@ -426,7 +432,7 @@ class XmlManifest(object):
     if name is None:
       s = m_url.rindex('/') + 1
       manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-      remote = _XmlRemote('origin', m_url[:s], manifestUrl)
+      remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
       name = m_url[s:]
 
     if name.endswith('.git'):
@@ -450,12 +456,15 @@ class XmlManifest(object):
     reads a <remote> element from the manifest file
     """
     name = self._reqatt(node, 'name')
+    alias = node.getAttribute('alias')
+    if alias == '':
+      alias = None
     fetch = self._reqatt(node, 'fetch')
     review = node.getAttribute('review')
     if review == '':
       review = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, fetch, manifestUrl, review)
+    return _XmlRemote(name, alias, fetch, manifestUrl, review)
 
   def _ParseDefault(self, node):
     """
@@ -565,8 +574,9 @@ class XmlManifest(object):
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
     groups = [x for x in re.split('[,\s]+', groups) if x]
-    if 'default' not in groups:
+
-      groups.append('default')
+    default_groups = ['default', 'name:%s' % name, 'path:%s' % path]
+    groups.extend(set(default_groups).difference(groups))
 
     if self.IsMirror:
       relpath = None

project.py

@@ -20,19 +20,9 @@ import random
 import re
 import shutil
 import stat
+import subprocess
 import sys
 import time
-import urllib2
-
-try:
-  import threading as _threading
-except ImportError:
-  import dummy_threading as _threading
-
-try:
-  from os import SEEK_END
-except ImportError:
-  SEEK_END = 2
 
 from color import Coloring
 from git_command import GitCommand
@@ -41,11 +31,10 @@ from error import DownloadError
 from error import GitError, HookError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
 from progress import Progress
+from trace import IsTrace, Trace
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
-_urllib_lock = _threading.Lock()
-
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -176,10 +165,11 @@ class ReviewableBranch(object):
       R_HEADS + self.name,
       '--')
 
-  def UploadForReview(self, people, auto_topic=False):
+  def UploadForReview(self, people, auto_topic=False, draft=False):
     self.project.UploadForReview(self.name,
                                  people,
-                                 auto_topic=auto_topic)
+                                 auto_topic=auto_topic,
+                                 draft=draft)
 
   def GetPublishedRefs(self):
     refs = {}
@@ -881,7 +871,8 @@ class Project(object):
 
   def UploadForReview(self, branch=None,
                       people=([],[]),
-                      auto_topic=False):
+                      auto_topic=False,
+                      draft=False):
     """Uploads the named branch for code review.
     """
     if branch is None:
@@ -920,7 +911,13 @@ class Project(object):
 
     if dest_branch.startswith(R_HEADS):
       dest_branch = dest_branch[len(R_HEADS):]
-    ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
+
+    upload_type = 'for'
+    if draft:
+      upload_type = 'drafts'
+
+    ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
+                                  dest_branch)
     if auto_topic:
       ref_spec = ref_spec + '/' + branch.name
     cmd.append(ref_spec)
@@ -1044,12 +1041,15 @@ class Project(object):
 
       if head == revid:
         # No changes; don't do anything further.
+        # Except if the head needs to be detached
         #
-        return
+        if not syncbuf.detach_head:
+          return
+      else:
+        lost = self._revlist(not_rev(revid), HEAD)
+        if lost:
+          syncbuf.info(self, "discarding %d commits", len(lost))
 
-      lost = self._revlist(not_rev(revid), HEAD)
-      if lost:
-        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(revid, quiet=True)
       except GitError, e:
@@ -1542,100 +1542,39 @@ class Project(object):
     return ok
 
   def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
-    keep = True
+    if os.path.exists(dstPath):
-    done = False
+      os.remove(dstPath)
-    dest = open(tmpPath, 'a+b')
-    try:
-      dest.seek(0, SEEK_END)
-      pos = dest.tell()
 
-      _urllib_lock.acquire()
+    cmd = ['curl', '--output', tmpPath, '--netrc', '--location']
-      try:
+    if quiet:
-        req = urllib2.Request(srcUrl)
+      cmd += ['--silent']
-        if pos > 0:
+    if os.path.exists(tmpPath):
-          req.add_header('Range', 'bytes=%d-' % pos)
+      size = os.stat(tmpPath).st_size
-
+      if size >= 1024:
-        try:
+        cmd += ['--continue-at', '%d' % (size,)]
-          r = urllib2.urlopen(req)
+      else:
-        except urllib2.HTTPError, e:
-          def _content_type():
-            try:
-              return e.info()['content-type']
-            except:
-              return None
-
-          if e.code in (401, 403, 404):
-            keep = False
-            return False
-          elif _content_type() == 'text/plain':
-            try:
-              msg = e.read()
-              if len(msg) > 0 and msg[-1] == '\n':
-                msg = msg[0:-1]
-              msg = ' (%s)' % msg
-            except:
-              msg = ''
-          else:
-            try:
-              from BaseHTTPServer import BaseHTTPRequestHandler
-              res = BaseHTTPRequestHandler.responses[e.code]
-              msg = ' (%s: %s)' % (res[0], res[1])
-            except:
-              msg = ''
-          raise DownloadError('HTTP %s%s' % (e.code, msg))
-        except urllib2.URLError, e:
-          raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
-      finally:
-        _urllib_lock.release()
-
-      p = None
-      try:
-        size = r.headers.get('content-length', 0)
-        unit = 1 << 10
-
-        if size and not quiet:
-          if size > 1024 * 1.3:
-            unit = 1 << 20
-            desc = 'MB'
-          else:
-            desc = 'KB'
-          p = Progress(
-            'Downloading %s' % self.relpath,
-            int(size) / unit,
-            units=desc)
-          if pos > 0:
-            p.update(pos / unit)
-
-        s = 0
-        while True:
-          d = r.read(8192)
-          if d == '':
-            done = True
-            return True
-          dest.write(d)
-          if p:
-            s += len(d)
-            if s >= unit:
-              p.update(s / unit)
-              s = s % unit
-        if p:
-          if s >= unit:
-            p.update(s / unit)
-          else:
-            p.update(1)
-      finally:
-        r.close()
-        if p:
-          p.end()
-    finally:
-      dest.close()
-
-      if os.path.exists(dstPath):
-        os.remove(dstPath)
-      if done:
-        os.rename(tmpPath, dstPath)
-      elif not keep:
         os.remove(tmpPath)
+    if 'http_proxy' in os.environ and 'darwin' == sys.platform:
+      cmd += ['--proxy', os.environ['http_proxy']]
+    cmd += [srcUrl]
+
+    if IsTrace():
+      Trace('%s', ' '.join(cmd))
+    try:
+      proc = subprocess.Popen(cmd)
+    except OSError:
+      return False
+
+    ok = proc.wait() == 0
+    if os.path.exists(tmpPath):
+      if ok and os.stat(tmpPath).st_size > 16:
+        os.rename(tmpPath, dstPath)
+        return True
+      else:
+        os.remove(tmpPath)
+        return False
+    else:
+      return False
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -2167,6 +2106,22 @@ class MetaProject(Project):
           self.revisionExpr = base
           self.revisionId = None
 
+  def MetaBranchSwitch(self, target):
+    """ Prepare MetaProject for manifest branch switch
+    """
+
+    # detach and delete manifest branch, allowing a new
+    # branch to take over
+    syncbuf = SyncBuffer(self.config, detach_head = True)
+    self.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
+    return GitCommand(self,
+                        ['update-ref', '-d', 'refs/heads/default'],
+                        capture_stdout = True,
+                        capture_stderr = True).Wait() == 0
+
+
   @property
   def LastFetch(self):
     try:

subcmds/init.py

@@ -187,6 +187,9 @@ to update the working directory files.
         shutil.rmtree(m.gitdir)
       sys.exit(1)
 
+    if opt.manifest_branch:
+      m.MetaBranchSwitch(opt.manifest_branch)
+
     syncbuf = SyncBuffer(m.config)
     m.Sync_LocalHalf(syncbuf)
     syncbuf.Finish()

subcmds/overview.py

@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+
+
+class Overview(PagedCommand):
+  common = True
+  helpSummary = "Display overview of unmerged project branches"
+  helpUsage = """
+%prog [--current-branch] [<project>...]
+"""
+  helpDescription = """
+The '%prog' command is used to display an overview of the projects branches,
+and list any local commits that have not yet been merged into the project.
+
+The -b/--current-branch option can be used to restrict the output to only
+branches currently checked out in each project.  By default, all branches
+are displayed.
+"""
+
+  def _Options(self, p):
+    p.add_option('-b', '--current-branch',
+                 dest="current_branch", action="store_true",
+                 help="Consider only checked out branches")
+
+  def Execute(self, opt, args):
+    all = []
+    for project in self.GetProjects(args):
+      br = [project.GetUploadableBranch(x)
+            for x in project.GetBranches().keys()]
+      br = [x for x in br if x]
+      if opt.current_branch:
+        br = [x for x in br if x.name == project.CurrentBranch]
+      all.extend(br)
+
+    if not all:
+      return
+
+    class Report(Coloring):
+      def __init__(self, config):
+        Coloring.__init__(self, config, 'status')
+        self.project = self.printer('header', attr='bold')
+
+    out = Report(all[0].project.config)
+    out.project('Projects Overview')
+    out.nl()
+
+    project = None
+
+    for branch in all:
+      if project != branch.project:
+        project = branch.project
+        out.nl()
+        out.project('project %s/' % project.relpath)
+        out.nl()
+
+      commits = branch.commits
+      date = branch.date
+      print '%s %-33s (%2d commit%s, %s)' % (
+            branch.name == project.CurrentBranch and '*' or ' ',
+            branch.name,
+            len(commits),
+            len(commits) != 1 and 's' or ' ',
+            date)
+      for commit in commits:
+        print '%-35s   - %s' % ('', commit)

subcmds/rebase.py

@@ -52,6 +52,9 @@ branch but need to incorporate new upstream changes "underneath" them.
     p.add_option('--whitespace',
                  dest='whitespace', action='store', metavar='WS',
                  help='Pass --whitespace to git rebase')
+    p.add_option('--auto-stash',
+                 dest='auto_stash', action='store_true',
+                 help='Stash local modifications before starting')
 
   def Execute(self, opt, args):
     all = self.GetProjects(args)
@@ -103,5 +106,23 @@ branch but need to incorporate new upstream changes "underneath" them.
       print >>sys.stderr, '# %s: rebasing %s -> %s' % \
         (project.relpath, cb, upbranch.LocalMerge)
 
+      needs_stash = False
+      if opt.auto_stash:
+        stash_args = ["update-index", "--refresh", "-q"]
+
+        if GitCommand(project, stash_args).Wait() != 0:
+          needs_stash = True
+          # Dirty index, requires stash...
+          stash_args = ["stash"]
+
+          if GitCommand(project, stash_args).Wait() != 0:
+            return -1
+
       if GitCommand(project, args).Wait() != 0:
         return -1
+
+      if needs_stash:
+        stash_args.append('pop')
+        stash_args.append('--quiet')
+        if GitCommand(project, stash_args).Wait() != 0:
+          return -1

subcmds/start.py

@@ -15,6 +15,7 @@
 
 import sys
 from command import Command
+from git_config import IsId
 from git_command import git
 from progress import Progress
 
@@ -56,6 +57,10 @@ revision specified in the manifest.
     pm = Progress('Starting %s' % nb, len(all))
     for project in all:
       pm.update()
+      # If the current revision is a specific SHA1 then we can't push back
+      # to it so substitute the manifest default revision instead.
+      if IsId(project.revisionExpr):
+        project.revisionExpr = self.manifest.default.revisionExpr
       if not project.StartBranch(nb):
         err.append(project)
     pm.end()

subcmds/sync.py

@@ -230,8 +230,10 @@ later is required to fix a server side protocol bug.
     if self.jobs == 1:
       for project in projects:
         pm.update()
-        if project.Sync_NetworkHalf(quiet=opt.quiet,
+        if project.Sync_NetworkHalf(
-                                    current_branch_only=opt.current_branch_only):
+            quiet=opt.quiet,
+            current_branch_only=opt.current_branch_only,
+            clone_bundle=not opt.no_clone_bundle):
           fetched.add(project.gitdir)
         else:
           print >>sys.stderr, 'error: Cannot fetch %s' % project.name

subcmds/upload.py

@@ -134,6 +134,9 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     p.add_option('--cbr', '--current-branch',
                  dest='current_branch', action='store_true',
                  help='Upload current git branch.')
+    p.add_option('-d', '--draft',
+                 action='store_true', dest='draft', default=False,
+                 help='If specified, upload as a draft.')
 
     # Options relating to upload hook.  Note that verify and no-verify are NOT
     # opposites of each other, which is why they store to different locations.
@@ -324,7 +327,7 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
            key = 'review.%s.uploadtopic' % branch.project.remote.review
            opt.auto_topic = branch.project.config.GetBoolean(key)
 
-        branch.UploadForReview(people, auto_topic=opt.auto_topic)
+        branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
         branch.uploaded = True
       except UploadError, e:
         branch.error = e