Fix ManifestParseError when first child node is comment

If the first line of manifest.xml is a XML comment, root.childNodes[0]
is not a 'manifest' element node. The python minidom module will makes
a 'Comment' node as root.childNodes[0]. Since the original code only
checks whether the first child node is 'manifest', it couldn't do any
command including 'sync' due to the 'ManifestParseError' exception. This
patch could allow the comments between '<?xml ...?>' and '<manifest>' in
the manifest.xml file.

Change-Id: I0b81dea4f806965eca90f704c8aa7df49c579402

SUBMITTING_PATCHES

@@ -5,7 +5,7 @@ Short Version:
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review:
 
-   git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/maint
+   git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master
 
 
 Long Version:
@@ -71,7 +71,7 @@ Push your patches over HTTPS to the review server, possibly through
 a remembered remote to make this easier in the future:
 
    git config remote.review.url https://gerrit-review.googlesource.com/git-repo
-   git config remote.review.push HEAD:refs/for/maint
+   git config remote.review.push HEAD:refs/for/master
 
    git push review
 

color.py

@@ -17,7 +17,6 @@ import os
 import sys
 
 import pager
-from git_config import GitConfig
 
 COLORS = {None     :-1,
           'normal' :-1,

command.py

@@ -70,7 +70,7 @@ class Command(object):
 
     groups = mp.config.GetString('manifest.groups')
     if not groups:
-      groups = 'default,platform-' + platform.system().lower()
+      groups = 'all,-notdefault,platform-' + platform.system().lower()
     groups = [x for x in re.split('[,\s]+', groups) if x]
 
     if not args:

docs/manifest-format.txt

@@ -125,21 +125,27 @@ Element manifest-server
 
 At most one manifest-server may be specified. The url attribute
 is used to specify the URL of a manifest server, which is an
-XML RPC service that will return a manifest in which each project
+XML RPC service.
-is pegged to a known good revision for the current branch and
-target.
 
-The manifest server should implement:
+The manifest server should implement the following RPC methods:
 
   GetApprovedManifest(branch, target)
 
+Return a manifest in which each project is pegged to a known good revision
+for the current branch and target.
+
 The target to use is defined by environment variables TARGET_PRODUCT
 and TARGET_BUILD_VARIANT. These variables are used to create a string
 of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug.
 If one of those variables or both are not present, the program will call
-GetApprovedManifest without the target paramater and the manifest server
+GetApprovedManifest without the target parameter and the manifest server
 should choose a reasonable default target.
 
+  GetManifest(tag)
+
+Return a manifest in which each project is pegged to the revision at
+the specified tag.
+
 
 Element project
 ---------------
@@ -156,7 +162,7 @@ URL to configure the Git remote with.  The URL gets formed as:
 
 where ${remote_fetch} is the remote's fetch attribute and
 ${project_name} is the project's name attribute.  The suffix ".git"
-is always appended as repo assumes the upstream is a forrest of
+is always appended as repo assumes the upstream is a forest of
 bare Git repositories.
 
 The project name must match the name Gerrit knows, if Gerrit is
@@ -178,11 +184,12 @@ the default element is used.
 
 Attribute `groups`: List of groups to which this project belongs,
 whitespace or comma separated.  All projects belong to the group
-"default", and each project automatically belongs to a group of
+"all", and each project automatically belongs to a group of
-it's name:`name` and path:`path`.  E.g. for
+its name:`name` and path:`path`.  E.g. for
 <project name="monkeys" path="barrel-of"/>, that project
 definition is implicitly in the following manifest groups:
-default, name:monkeys, and path:barrel-of.
+default, name:monkeys, and path:barrel-of.  If you place a project in the
+group "notdefault", it will not be automatically downloaded by repo.
 
 Element annotation
 ------------------

git_refs.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import os
-import sys
 from trace import Trace
 
 HEAD    = 'HEAD'

main.py

@@ -35,7 +35,6 @@ from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
-from command import PagedCommand
 from subcmds.version import Version
 from editor import Editor
 from error import DownloadError

manifest_xml.py

@@ -20,7 +20,7 @@ import sys
 import urlparse
 import xml.dom.minidom
 
-from git_config import GitConfig, IsId
+from git_config import GitConfig
 from project import RemoteSpec, Project, MetaProject, R_HEADS, HEAD
 from error import ManifestParseError
 
@@ -130,7 +130,7 @@ class XmlManifest(object):
 
     groups = mp.config.GetString('manifest.groups')
     if not groups:
-      groups = 'default'
+      groups = 'all'
     groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     doc = xml.dom.minidom.Document()
@@ -211,7 +211,8 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
-      egroups = [g for g in p.groups if g != 'default']
+      default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
+      egroups = [g for g in p.groups if g not in default_groups]
       if egroups:
         e.setAttribute('groups', ','.join(egroups))
 
@@ -308,12 +309,14 @@ class XmlManifest(object):
     if not root or not root.childNodes:
       raise ManifestParseError("no root node in %s" % (path,))
 
-    config = root.childNodes[0]
+    for manifest in root.childNodes:
-    if config.nodeName != 'manifest':
+      if manifest.nodeName == 'manifest':
+        break
+    else:
       raise ManifestParseError("no <manifest> in %s" % (path,))
 
     nodes = []
-    for node in config.childNodes:
+    for node in manifest.childNodes:
         if node.nodeName == 'include':
             name = self._reqatt(node, 'name')
             fp = os.path.join(include_root, name)
@@ -432,7 +435,7 @@ class XmlManifest(object):
     if name is None:
       s = m_url.rindex('/') + 1
       manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-      remote = _XmlRemote('origin', m_url[:s], manifestUrl)
+      remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
       name = m_url[s:]
 
     if name.endswith('.git'):

project.py

@@ -20,32 +20,19 @@ import random
 import re
 import shutil
 import stat
+import subprocess
 import sys
 import time
-import urllib2
-
-try:
-  import threading as _threading
-except ImportError:
-  import dummy_threading as _threading
-
-try:
-  from os import SEEK_END
-except ImportError:
-  SEEK_END = 2
 
 from color import Coloring
 from git_command import GitCommand
 from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
-from error import DownloadError
+from error import GitError, HookError, UploadError
-from error import GitError, HookError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
-from progress import Progress
+from trace import IsTrace, Trace
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
-_urllib_lock = _threading.Lock()
-
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
@@ -91,21 +78,6 @@ def _ProjectHooks():
     _project_hook_list = map(lambda x: os.path.join(d, x), os.listdir(d))
   return _project_hook_list
 
-def relpath(dst, src):
-  src = os.path.dirname(src)
-  top = os.path.commonprefix([dst, src])
-  if top.endswith('/'):
-    top = top[:-1]
-  else:
-    top = os.path.dirname(top)
-
-  tmp = src
-  rel = ''
-  while top != tmp:
-    rel += '../'
-    tmp = os.path.dirname(tmp)
-  return rel + dst[len(top) + 1:]
-
 
 class DownloadedChange(object):
   _commit_cache = None
@@ -660,20 +632,21 @@ class Project(object):
     """Returns true if the manifest groups specified at init should cause
        this project to be synced.
        Prefixing a manifest group with "-" inverts the meaning of a group.
-       All projects are implicitly labelled with "default".
+       All projects are implicitly labelled with "all".
 
        labels are resolved in order.  In the example case of
-       project_groups: "default,group1,group2"
+       project_groups: "all,group1,group2"
        manifest_groups: "-group1,group2"
        the project will be matched.
     """
-    if self.groups is None:
+    expanded_manifest_groups = manifest_groups or ['all', '-notdefault']
-      return True
+    expanded_project_groups = ['all'] + (self.groups or [])
+
     matched = False
-    for group in manifest_groups:
+    for group in expanded_manifest_groups:
-      if group.startswith('-') and group[1:] in self.groups:
+      if group.startswith('-') and group[1:] in expanded_project_groups:
         matched = False
-      elif group in self.groups:
+      elif group in expanded_project_groups:
         matched = True
 
     return matched
@@ -1553,100 +1526,49 @@ class Project(object):
     return ok
 
   def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
-    keep = True
-    done = False
-    dest = open(tmpPath, 'a+b')
-    try:
-      dest.seek(0, SEEK_END)
-      pos = dest.tell()
-
-      _urllib_lock.acquire()
-      try:
-        req = urllib2.Request(srcUrl)
-        if pos > 0:
-          req.add_header('Range', 'bytes=%d-' % (pos,))
-
-        try:
-          r = urllib2.urlopen(req)
-        except urllib2.HTTPError, e:
-          def _content_type():
-            try:
-              return e.info()['content-type']
-            except:
-              return None
-
-          if e.code in (401, 403, 404):
-            keep = False
-            return False
-          elif _content_type() == 'text/plain':
-            try:
-              msg = e.read()
-              if len(msg) > 0 and msg[-1] == '\n':
-                msg = msg[0:-1]
-              msg = ' (%s)' % (msg,)
-            except:
-              msg = ''
-          else:
-            try:
-              from BaseHTTPServer import BaseHTTPRequestHandler
-              res = BaseHTTPRequestHandler.responses[e.code]
-              msg = ' (%s: %s)' % (res[0], res[1])
-            except:
-              msg = ''
-          raise DownloadError('HTTP %s%s' % (e.code, msg))
-        except urllib2.URLError, e:
-          raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
-      finally:
-        _urllib_lock.release()
-
-      p = None
-      try:
-        size = pos + r.headers.get('content-length', 0)
-        unit = 1 << 10
-
-        if size and not quiet:
-          if size > 1024 * 1.3:
-            unit = 1 << 20
-            desc = 'MB'
-          else:
-            desc = 'KB'
-          p = Progress(
-            'Downloading %s' % (self.relpath,),
-            int(size) / unit,
-            units=desc)
-          if pos > 0:
-            p.update(pos / unit)
-
-        s = 0
-        while True:
-          d = r.read(8192)
-          if d == '':
-            done = True
-            return True
-          dest.write(d)
-          if p:
-            s += len(d)
-            if s >= unit:
-              p.update(s / unit)
-              s = s % unit
-        if p:
-          if s >= unit:
-            p.update(s / unit)
-          else:
-            p.update(1)
-      finally:
-        r.close()
-        if p:
-          p.end()
-    finally:
-      dest.close()
-
     if os.path.exists(dstPath):
       os.remove(dstPath)
-      if done:
+
-        os.rename(tmpPath, dstPath)
+    cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
-      elif not keep:
+    if quiet:
+      cmd += ['--silent']
+    if os.path.exists(tmpPath):
+      size = os.stat(tmpPath).st_size
+      if size >= 1024:
+        cmd += ['--continue-at', '%d' % (size,)]
+      else:
         os.remove(tmpPath)
+    if 'http_proxy' in os.environ and 'darwin' == sys.platform:
+      cmd += ['--proxy', os.environ['http_proxy']]
+    cmd += [srcUrl]
+
+    if IsTrace():
+      Trace('%s', ' '.join(cmd))
+    try:
+      proc = subprocess.Popen(cmd)
+    except OSError:
+      return False
+
+    curlret = proc.wait()
+
+    if curlret == 22:
+      # From curl man page:
+      # 22: HTTP page not retrieved. The requested url was not found or
+      # returned another error with the HTTP error code being 400 or above.
+      # This return code only appears if -f, --fail is used.
+      if not quiet:
+        print >> sys.stderr, "Server does not provide clone.bundle; ignoring."
+      return False
+
+    if os.path.exists(tmpPath):
+      if curlret == 0 and os.stat(tmpPath).st_size > 16:
+        os.rename(tmpPath, dstPath)
+        return True
+      else:
+        os.remove(tmpPath)
+        return False
+    else:
+      return False
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -1769,7 +1691,7 @@ class Project(object):
           _error("%s: Not replacing %s hook", self.relpath, name)
           continue
       try:
-        os.symlink(relpath(stock_hook, dst), dst)
+        os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
       except OSError, e:
         if e.errno == errno.EPERM:
           raise GitError('filesystem must support symlinks')
@@ -1830,7 +1752,7 @@ class Project(object):
           src = os.path.join(self.gitdir, name)
           dst = os.path.join(dotgit, name)
           if os.path.islink(dst) or not os.path.exists(dst):
-            os.symlink(relpath(src, dst), dst)
+            os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
           else:
             raise GitError('cannot overwrite a local work tree')
         except OSError, e:

subcmds/cherry_pick.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys, re, string, random, os
+import sys, re
 from command import Command
 from git_command import GitCommand
 

subcmds/download.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import re
 import sys
 

subcmds/grep.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import sys
-from optparse import SUPPRESS_HELP
 from color import Coloring
 from command import PagedCommand
 from git_command import git_require, GitCommand

subcmds/init.py

@@ -81,7 +81,8 @@ to update the working directory files.
                  help='initial manifest file', metavar='NAME.xml')
     g.add_option('--mirror',
                  dest='mirror', action='store_true',
-                 help='mirror the forrest')
+                 help='create a replica of the remote repositories '
+                      'rather than a client working directory')
     g.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
@@ -89,12 +90,12 @@ to update the working directory files.
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
     g.add_option('-g', '--groups',
-                 dest='groups', default='default',
+                 dest='groups', default='all,-notdefault',
                  help='restrict manifest projects to ones with a specified group',
                  metavar='GROUP')
     g.add_option('-p', '--platform',
                  dest='platform', default='auto',
-                 help='restrict manifest projects to ones with a specified'
+                 help='restrict manifest projects to ones with a specified '
                       'platform group [auto|all|none|linux|darwin|...]',
                  metavar='PLATFORM')
 
@@ -163,7 +164,7 @@ to update the working directory files.
 
     groups = [x for x in groups if x]
     groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
+    if opt.platform == 'auto' and groupstr == 'all,-notdefault,platform-' + platform.system().lower():
       groupstr = None
     m.config.SetString('manifest.groups', groupstr)
 

subcmds/manifest.py

@@ -50,6 +50,7 @@ in a Git repository for use during future 'repo init' invocations.
                  help='Save revisions as current HEAD')
     p.add_option('-o', '--output-file',
                  dest='output_file',
+                 default='-',
                  help='File to save the manifest to',
                  metavar='-|NAME.xml')
 

subcmds/rebase.py

@@ -17,8 +17,6 @@ import sys
 
 from command import Command
 from git_command import GitCommand
-from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
-from error import GitError
 
 class Rebase(Command):
   common = True

subcmds/status.py

@@ -122,6 +122,7 @@ the following meanings:
         t = _threading.Thread(target=self._StatusHelper,
                               args=(project, counter, sem, output))
         threads_and_output.append((t, output))
+        t.daemon = True
         t.start()
       for (t, output) in threads_and_output:
         t.join()

subcmds/sync.py

@@ -37,13 +37,11 @@ except ImportError:
     return (256, 256)
 
 from git_command import GIT
-from git_refs import R_HEADS
+from git_refs import R_HEADS, HEAD
-from project import HEAD
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
-from project import R_HEADS
 from project import SyncBuffer
 from progress import Progress
 
@@ -230,8 +228,10 @@ later is required to fix a server side protocol bug.
     if self.jobs == 1:
       for project in projects:
         pm.update()
-        if project.Sync_NetworkHalf(quiet=opt.quiet,
+        if project.Sync_NetworkHalf(
-                                    current_branch_only=opt.current_branch_only):
+            quiet=opt.quiet,
+            current_branch_only=opt.current_branch_only,
+            clone_bundle=not opt.no_clone_bundle):
           fetched.add(project.gitdir)
         else:
           print >>sys.stderr, 'error: Cannot fetch %s' % project.name
@@ -259,6 +259,8 @@ later is required to fix a server side protocol bug.
                                       pm,
                                       sem,
                                       err_event))
+        # Ensure that Ctrl-C will not freeze the repo process.
+        t.daemon = True
         threads.add(t)
         t.start()
 
@@ -402,9 +404,13 @@ uncommitted changes are present' % project.relpath
         else:
           print >>sys.stderr, 'error: %s' % manifest_str
           sys.exit(1)
-      except socket.error:
+      except (socket.error, IOError, xmlrpclib.Fault), e:
-        print >>sys.stderr, 'error: cannot connect to manifest server %s' % (
+        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
-            self.manifest.manifest_server)
+            self.manifest.manifest_server, e)
+        sys.exit(1)
+      except xmlrpclib.ProtocolError, e:
+        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
+            self.manifest.manifest_server, e.errcode, e.errmsg)
         sys.exit(1)
 
     rp = self.manifest.repoProject