Remove import_tar, import_zip and the <snapshot> elements

Now that repo relies only on the git data stream (as it is much
faster to download through) we don't really need to be parsing the
<snapshot> elements within manifest.  Its a lot of complex code to
convert the tar (or zip) through to a fast import stream, and we
just aren't calling it anymore.

Signed-off-by: Shawn O. Pearce <sop@google.com>

gerrit_upload.py

@@ -15,7 +15,6 @@
 
 import getpass
 import os
-import subprocess
 import sys
 from tempfile import mkstemp
 

git_config.py

@@ -28,9 +28,13 @@ def IsId(rev):
 
 
 class GitConfig(object):
+  _ForUser = None
+
   @classmethod
   def ForUser(cls):
-    return cls(file = os.path.expanduser('~/.gitconfig'))
+    if cls._ForUser is None:
+      cls._ForUser = cls(file = os.path.expanduser('~/.gitconfig'))
+    return cls._ForUser
 
   @classmethod
   def ForRepository(cls, gitdir, defaults=None):

import_ext.py

@@ -1,422 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import random
-import stat
-import sys
-import urllib2
-import StringIO
-
-from error import GitError, ImportError
-from git_command import GitCommand
-
-class ImportExternal(object):
-  """Imports a single revision from a non-git data source.
-     Suitable for use to import a tar or zip based snapshot.
-  """
-  def __init__(self):
-    self._marks = 0
-    self._files = {}
-    self._tempref = 'refs/repo-external/import'
-
-    self._urls = []
-    self._remap = []
-    self.parent = None
-    self._user_name = 'Upstream'
-    self._user_email = 'upstream-import@none'
-    self._user_when = 1000000
-
-    self.commit = None
-
-  def Clone(self):
-    r = self.__class__()
-
-    r.project = self.project
-    for u in self._urls:
-      r._urls.append(u)
-    for p in self._remap:
-      r._remap.append(_PathMap(r, p._old, p._new))
-
-    return r
-
-  def SetProject(self, project):
-    self.project = project
-
-  def SetVersion(self, version):
-    self.version = version
-
-  def AddUrl(self, url):
-    self._urls.append(url)
-
-  def SetParent(self, commit_hash):
-    self.parent = commit_hash
-
-  def SetCommit(self, commit_hash):
-    self.commit = commit_hash
-
-  def RemapPath(self, old, new, replace_version=True):
-    self._remap.append(_PathMap(self, old, new))
-
-  @property
-  def TagName(self):
-    v = ''
-    for c in self.version:
-      if c >= '0' and c <= '9':
-        v += c
-      elif c >= 'A' and c <= 'Z':
-        v += c
-      elif c >= 'a' and c <= 'z':
-        v += c
-      elif c in ('-', '_', '.', '/', '+', '@'):
-        v += c
-    return 'upstream/%s' % v
-
-  @property
-  def PackageName(self):
-    n = self.project.name
-    if n.startswith('platform/'):
-      # This was not my finest moment...
-      #
-      n = n[len('platform/'):]
-    return n
-
-  def Import(self):
-    self._need_graft = False
-    if self.parent:
-      try:
-        self.project.bare_git.cat_file('-e', self.parent)
-      except GitError:
-        self._need_graft = True
-
-    gfi = GitCommand(self.project,
-                     ['fast-import', '--force', '--quiet'],
-                     bare = True,
-                     provide_stdin = True)
-    try:
-      self._out = gfi.stdin
-
-      try:
-        self._UnpackFiles()
-        self._MakeCommit()
-        self._out.flush()
-      finally:
-        rc = gfi.Wait()
-      if rc != 0:
-        raise ImportError('fast-import failed')
-
-      if self._need_graft:
-        id = self._GraftCommit()
-      else:
-        id = self.project.bare_git.rev_parse('%s^0' % self._tempref)
-
-      if self.commit and self.commit != id:
-        raise ImportError('checksum mismatch: %s expected,'
-                          ' %s imported' % (self.commit, id))
-
-      self._MakeTag(id)
-      return id
-    finally:
-      try:
-        self.project.bare_git.DeleteRef(self._tempref)
-      except GitError:
-        pass
-
-  def _PickUrl(self, failed):
-    u = map(lambda x: x.replace('%version%', self.version), self._urls)
-    for f in failed:
-      if f in u:
-        u.remove(f)
-    if len(u) == 0:
-      return None
-    return random.choice(u)
-
-  def _OpenUrl(self):
-    failed = {}
-    while True:
-      url = self._PickUrl(failed.keys())
-      if url is None:
-        why = 'Cannot download %s' % self.project.name
-
-        if failed:
-          why += ': one or more mirrors are down\n'
-          bad_urls = list(failed.keys())
-          bad_urls.sort()
-          for url in bad_urls:
-            why += '  %s: %s\n' % (url, failed[url])
-        else:
-          why += ': no mirror URLs'
-        raise ImportError(why)
-
-      print >>sys.stderr, "Getting %s ..." % url
-      try:
-        return urllib2.urlopen(url), url
-      except urllib2.HTTPError, e:
-        failed[url] = e.code
-      except urllib2.URLError, e:
-        failed[url] = e.reason[1]
-      except OSError, e:
-        failed[url] = e.strerror
-
-  def _UnpackFiles(self):
-    raise NotImplementedError
-
-  def _NextMark(self):
-    self._marks += 1
-    return self._marks
-
-  def _UnpackOneFile(self, mode, size, name, fd):
-    if stat.S_ISDIR(mode):    # directory
-      return
-    else:
-      mode = self._CleanMode(mode, name)
-
-    old_name = name
-    name = self._CleanName(name)
-
-    if stat.S_ISLNK(mode) and self._remap:
-      # The link is relative to the old_name, and may need to
-      # be rewritten according to our remap rules if it goes
-      # up high enough in the tree structure.
-      #
-      dest = self._RewriteLink(fd.read(size), old_name, name)
-      fd = StringIO.StringIO(dest)
-      size = len(dest)
-
-    fi = _File(mode, name, self._NextMark())
-
-    self._out.write('blob\n')
-    self._out.write('mark :%d\n' % fi.mark)
-    self._out.write('data %d\n' % size)
-    while size > 0:
-      n = min(2048, size)
-      self._out.write(fd.read(n))
-      size -= n
-    self._out.write('\n')
-    self._files[fi.name] = fi
-
-  def _SetFileMode(self, name, mode):
-    if not stat.S_ISDIR(mode):
-      mode = self._CleanMode(mode, name)
-      name = self._CleanName(name)
-      try:
-        fi = self._files[name]
-      except KeyError:
-        raise ImportError('file %s was not unpacked' % name)
-      fi.mode = mode
-
-  def _RewriteLink(self, dest, relto_old, relto_new):
-    # Drop the last components of the symlink itself
-    # as the dest is relative to the directory its in.
-    #
-    relto_old = _TrimPath(relto_old)
-    relto_new = _TrimPath(relto_new)
-
-    # Resolve the link to be absolute from the top of
-    # the archive, so we can remap its destination.
-    #
-    while dest.find('/./') >= 0 or dest.find('//') >= 0:
-      dest = dest.replace('/./', '/')
-      dest = dest.replace('//', '/')
-
-    if dest.startswith('../') or dest.find('/../') > 0:
-      dest = _FoldPath('%s/%s' % (relto_old, dest))
-
-    for pm in self._remap:
-      if pm.Matches(dest):
-        dest = pm.Apply(dest)
-        break
-
-    dest, relto_new = _StripCommonPrefix(dest, relto_new)
-    while relto_new:
-      i = relto_new.find('/')
-      if i > 0:
-        relto_new = relto_new[i + 1:]
-      else:
-        relto_new = ''
-      dest = '../' + dest
-    return dest
-
-  def _CleanMode(self, mode, name):
-    if stat.S_ISREG(mode):  # regular file
-      if (mode & 0111) == 0:
-        return 0644
-      else:
-        return 0755
-    elif stat.S_ISLNK(mode):  # symlink
-      return stat.S_IFLNK
-    else:
-      raise ImportError('invalid mode %o in %s' % (mode, name))
-
-  def _CleanName(self, name):
-    old_name = name
-    for pm in self._remap:
-      if pm.Matches(name):
-        name = pm.Apply(name)
-        break
-    while name.startswith('/'):
-      name = name[1:]
-    if not name:
-      raise ImportError('path %s is empty after remap' % old_name)
-    if name.find('/./') >= 0 or name.find('/../') >= 0:
-      raise ImportError('path %s contains relative parts' % name)
-    return name
-
-  def _MakeCommit(self):
-    msg = '%s %s\n' % (self.PackageName, self.version)
-
-    self._out.write('commit %s\n' % self._tempref)
-    self._out.write('committer %s <%s> %d +0000\n' % (
-                    self._user_name,
-                    self._user_email,
-                    self._user_when))
-    self._out.write('data %d\n' % len(msg))
-    self._out.write(msg)
-    self._out.write('\n')
-    if self.parent and not self._need_graft:
-      self._out.write('from %s^0\n' % self.parent)
-      self._out.write('deleteall\n')
-
-    for f in self._files.values():
-      self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name))
-    self._out.write('\n')
-
-  def _GraftCommit(self):
-    raw = self.project.bare_git.cat_file('commit', self._tempref)
-    raw = raw.split("\n")
-    while raw[1].startswith('parent '):
-      del raw[1]
-    raw.insert(1, 'parent %s' % self.parent)
-    id = self._WriteObject('commit', "\n".join(raw))
-
-    graft_file = os.path.join(self.project.gitdir, 'info/grafts')
-    if os.path.exists(graft_file):
-      graft_list = open(graft_file, 'rb').read().split("\n")
-      if graft_list and graft_list[-1] == '':
-        del graft_list[-1]
-    else:
-      graft_list = []
-
-    exists = False
-    for line in graft_list:
-      if line == id:
-        exists = True
-        break
-
-    if not exists:
-      graft_list.append(id)
-      graft_list.append('')
-      fd = open(graft_file, 'wb')
-      fd.write("\n".join(graft_list))
-      fd.close()
-
-    return id
-
-  def _MakeTag(self, id):
-    name = self.TagName
-
-    raw = []
-    raw.append('object %s' % id)
-    raw.append('type commit')
-    raw.append('tag %s' % name)
-    raw.append('tagger %s <%s> %d +0000' % (
-      self._user_name,
-      self._user_email,
-      self._user_when))
-    raw.append('')
-    raw.append('%s %s\n' % (self.PackageName, self.version))
-
-    tagid = self._WriteObject('tag', "\n".join(raw))
-    self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid)
-
-  def _WriteObject(self, type, data):
-    wo = GitCommand(self.project,
-                    ['hash-object', '-t', type, '-w', '--stdin'],
-                    bare = True,
-                    provide_stdin = True,
-                    capture_stdout = True,
-                    capture_stderr = True)
-    wo.stdin.write(data)
-    if wo.Wait() != 0:
-      raise GitError('cannot create %s from (%s)' % (type, data))
-    return wo.stdout[:-1]
-
-
-def _TrimPath(path):
-  i = path.rfind('/')
-  if i > 0:
-    path = path[0:i]
-  return ''
-
-def _StripCommonPrefix(a, b):
-  while True:
-    ai = a.find('/')
-    bi = b.find('/')
-    if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]:
-      a = a[ai + 1:]
-      b = b[bi + 1:]
-    else:
-      break
-  return a, b
-
-def _FoldPath(path):
-  while True:
-    if path.startswith('../'):
-      return path
-
-    i = path.find('/../')
-    if i <= 0:
-      if path.startswith('/'):
-        return path[1:]
-      return path
-
-    lhs = path[0:i]
-    rhs = path[i + 4:]
-
-    i = lhs.rfind('/')
-    if i > 0:
-      path = lhs[0:i + 1] + rhs
-    else:
-      path = rhs
-
-class _File(object):
-  def __init__(self, mode, name, mark):
-    self.mode = mode
-    self.name = name
-    self.mark = mark
-
-
-class _PathMap(object):
-  def __init__(self, imp, old, new):
-    self._imp = imp
-    self._old = old
-    self._new = new
-
-  def _r(self, p):
-    return p.replace('%version%', self._imp.version)
-
-  @property
-  def old(self):
-    return self._r(self._old)
-
-  @property
-  def new(self):
-    return self._r(self._new)
-
-  def Matches(self, name):
-    return name.startswith(self.old)
-
-  def Apply(self, name):
-    return self.new + name[len(self.old):]

import_tar.py

@@ -1,206 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import bz2
-import stat
-import tarfile
-import zlib
-import StringIO
-
-from import_ext import ImportExternal
-from error import ImportError
-
-class ImportTar(ImportExternal):
-  """Streams a (optionally compressed) tar file from the network
-     directly into a Project's Git repository.
-  """
-  @classmethod
-  def CanAccept(cls, url):
-    """Can this importer read and unpack the data stored at url?
-    """
-    if url.endswith('.tar.gz') or url.endswith('.tgz'):
-      return True
-    if url.endswith('.tar.bz2'):
-      return True
-    if url.endswith('.tar'):
-      return True
-    return False
-
-  def _UnpackFiles(self):
-    url_fd, url = self._OpenUrl()
-    try:
-      if url.endswith('.tar.gz') or url.endswith('.tgz'):
-        tar_fd = _Gzip(url_fd)
-      elif url.endswith('.tar.bz2'):
-        tar_fd = _Bzip2(url_fd)
-      elif url.endswith('.tar'):
-        tar_fd = _Raw(url_fd)
-      else:
-        raise ImportError('non-tar file extension: %s' % url)
-
-      try:
-        tar = tarfile.TarFile(name = url,
-                              mode = 'r',
-                              fileobj = tar_fd)
-        try:
-          for entry in tar:
-            mode = entry.mode
-
-            if (mode & 0170000) == 0:
-              if entry.isdir():
-                mode |= stat.S_IFDIR
-              elif entry.isfile() or entry.islnk():  # hard links as files
-                mode |= stat.S_IFREG
-              elif entry.issym():
-                mode |= stat.S_IFLNK
-
-            if stat.S_ISLNK(mode):   # symlink
-              data_fd = StringIO.StringIO(entry.linkname)
-              data_sz = len(entry.linkname)
-            elif stat.S_ISDIR(mode): # directory
-              data_fd = StringIO.StringIO('')
-              data_sz = 0
-            else:
-              data_fd = tar.extractfile(entry)
-              data_sz = entry.size
-
-            self._UnpackOneFile(mode, data_sz, entry.name, data_fd)
-        finally:
-          tar.close()
-      finally:
-        tar_fd.close()
-    finally:
-      url_fd.close()
-
-
-
-class _DecompressStream(object):
-  """file like object to decompress a tar stream
-  """
-  def __init__(self, fd):
-    self._fd = fd
-    self._pos = 0
-    self._buf = None
-
-  def tell(self):
-    return self._pos
-
-  def seek(self, offset):
-    d = offset - self._pos
-    if d > 0:
-      self.read(d)
-    elif d == 0:
-      pass
-    else:
-      raise NotImplementedError, 'seek backwards'
-
-  def close(self):
-    self._fd = None
-
-  def read(self, size = -1):
-    if not self._fd:
-      raise EOFError, 'Reached EOF'
-    
-    r = []
-    try:
-      if size >= 0:
-        self._ReadChunk(r, size)
-      else:
-        while True:
-          self._ReadChunk(r, 2048)
-    except EOFError:
-      pass
-
-    if len(r) == 1:
-      r = r[0]
-    else:
-      r = ''.join(r)
-    self._pos += len(r)
-    return r
-
-  def _ReadChunk(self, r, size):
-    b = self._buf
-    try:
-      while size > 0:
-        if b is None or len(b) == 0:
-          b = self._Decompress(self._fd.read(2048))
-          continue
-
-        use = min(size, len(b))
-        r.append(b[:use])
-        b = b[use:]
-        size -= use
-    finally:
-      self._buf = b
-
-  def _Decompress(self, b):
-    raise NotImplementedError, '_Decompress'
-
-
-class _Raw(_DecompressStream):
-  """file like object for an uncompressed stream
-  """
-  def __init__(self, fd):
-    _DecompressStream.__init__(self, fd)
-
-  def _Decompress(self, b):
-    return b
-
-
-class _Bzip2(_DecompressStream):
-  """file like object to decompress a .bz2 stream
-  """
-  def __init__(self, fd):
-    _DecompressStream.__init__(self, fd)
-    self._bz = bz2.BZ2Decompressor()
-
-  def _Decompress(self, b):
-    return self._bz.decompress(b)
-
-
-_FHCRC, _FEXTRA, _FNAME, _FCOMMENT = 2, 4, 8, 16
-class _Gzip(_DecompressStream):
-  """file like object to decompress a .gz stream
-  """
-  def __init__(self, fd):
-    _DecompressStream.__init__(self, fd)
-    self._z = zlib.decompressobj(-zlib.MAX_WBITS)
-
-    magic = fd.read(2)
-    if magic != '\037\213':
-      raise IOError, 'Not a gzipped file'
-
-    method = ord(fd.read(1))
-    if method != 8:
-      raise IOError, 'Unknown compression method'
-
-    flag = ord(fd.read(1))
-    fd.read(6)
-
-    if flag & _FEXTRA:
-      xlen = ord(fd.read(1))
-      xlen += 256 * ord(fd.read(1))
-      fd.read(xlen)
-    if flag & _FNAME:
-      while fd.read(1) != '\0':
-        pass
-    if flag & _FCOMMENT:
-      while fd.read(1) != '\0':
-        pass
-    if flag & _FHCRC:
-      fd.read(2)
-
-  def _Decompress(self, b):
-    return self._z.decompress(b)

import_zip.py

@@ -1,345 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import stat
-import struct
-import zlib
-import cStringIO
-
-from import_ext import ImportExternal
-from error import ImportError
-
-class ImportZip(ImportExternal):
-  """Streams a zip file from the network directly into a Project's
-     Git repository.
-  """
-  @classmethod
-  def CanAccept(cls, url):
-    """Can this importer read and unpack the data stored at url?
-    """
-    if url.endswith('.zip') or url.endswith('.jar'):
-      return True
-    return False
-
-  def _UnpackFiles(self):
-    url_fd, url = self._OpenUrl()
-    try:
-      if not self.__class__.CanAccept(url):
-        raise ImportError('non-zip file extension: %s' % url)
-
-      zip = _ZipFile(url_fd)
-      for entry in zip.FileRecords():
-        data = zip.Open(entry).read()
-        sz = len(data)
-
-        if data and _SafeCRLF(data):
-          data = data.replace('\r\n', '\n')
-          sz = len(data)
-
-        fd = cStringIO.StringIO(data)
-        self._UnpackOneFile(entry.mode, sz, entry.name, fd)
-        zip.Close(entry)
-
-      for entry in zip.CentralDirectory():
-        self._SetFileMode(entry.name, entry.mode)
-
-      zip.CheckTail()
-    finally:
-      url_fd.close()
-
-
-def _SafeCRLF(data):
-  """Is it reasonably safe to perform a CRLF->LF conversion?
-
-     If the stream contains a NUL byte it is likely binary,
-     and thus a CRLF->LF conversion may damage the stream.
-
-     If the only NUL is in the last position of the stream,
-     but it otherwise can do a CRLF<->LF conversion we do
-     the CRLF conversion anyway.  At least one source ZIP
-     file has this structure in its source code.
-
-     If every occurrance of a CR and LF is paired up as a
-     CRLF pair then the conversion is safely bi-directional.
-     s/\r\n/\n/g == s/\n/\r\\n/g can convert between them.
-  """
-  nul = data.find('\0')
-  if 0 <= nul and nul < (len(data) - 1):
-    return False
-
-  n_lf = 0
-  last = 0
-  while True:
-    lf = data.find('\n', last)
-    if lf < 0:
-      break
-    if lf == 0 or data[lf - 1] != '\r':
-      return False
-    last = lf + 1
-    n_lf += 1
-  return n_lf > 0
-
-class _ZipFile(object):
-  """Streaming iterator to parse a zip file on the fly.
-  """
-  def __init__(self, fd):
-    self._fd = _UngetStream(fd)
-
-  def FileRecords(self):
-    return _FileIter(self._fd)
-
-  def CentralDirectory(self):
-    return _CentIter(self._fd)
-
-  def CheckTail(self):
-    type_buf = self._fd.read(4)
-    type = struct.unpack('<I', type_buf)[0]
-    if type != 0x06054b50:  # end of central directory
-      raise ImportError('zip record %x unsupported' % type)
-
-  def Open(self, entry):
-    if entry.is_compressed:
-      return _InflateStream(self._fd)
-    else:
-      if entry.has_trailer:
-        raise ImportError('unable to extract streamed zip')
-      return _FixedLengthStream(self._fd, entry.uncompressed_size)
-
-  def Close(self, entry):
-    if entry.has_trailer:
-      type = struct.unpack('<I', self._fd.read(4))[0]
-      if type == 0x08074b50:
-        # Not a formal type marker, but commonly seen in zips
-        # as the data descriptor signature.
-        #
-        struct.unpack('<3I', self._fd.read(12))
-      else:
-        # No signature for the data descriptor, so read the
-        # remaining fields out of the stream
-        #
-        self._fd.read(8)
-
-
-class _FileIter(object):
-  def __init__(self, fd):
-    self._fd = fd
-
-  def __iter__(self):
-    return self
-
-  def next(self):
-    fd = self._fd
-
-    type_buf = fd.read(4)
-    type = struct.unpack('<I', type_buf)[0]
-
-    if type != 0x04034b50:    # local file header
-      fd.unread(type_buf)
-      raise StopIteration()
-
-    rec = _FileHeader(fd.read(26))
-    rec.name = fd.read(rec.name_len)
-    fd.read(rec.extra_len)
-
-    if rec.name.endswith('/'):
-      rec.name = rec.name[:-1]
-      rec.mode = stat.S_IFDIR | 0777
-    return rec
-
-
-class _FileHeader(object):
-  """Information about a single file in the archive.
-     0  version needed to extract       2 bytes
-     1  general purpose bit flag        2 bytes
-     2  compression method              2 bytes
-     3  last mod file time              2 bytes
-     4  last mod file date              2 bytes
-     5  crc-32                          4 bytes
-     6  compressed size                 4 bytes
-     7  uncompressed size               4 bytes
-     8  file name length                2 bytes
-     9  extra field length              2 bytes
-  """
-  def __init__(self, raw_bin):
-    rec = struct.unpack('<5H3I2H', raw_bin)
-    
-    if rec[2] == 8:
-      self.is_compressed = True
-    elif rec[2] == 0:
-      self.is_compressed = False
-    else:
-      raise ImportError('unrecognized compression format')
-
-    if rec[1] & (1 << 3):
-      self.has_trailer = True
-    else:
-      self.has_trailer = False
-
-    self.compressed_size  = rec[6]
-    self.uncompressed_size = rec[7]
-    self.name_len = rec[8]
-    self.extra_len = rec[9]
-    self.mode = stat.S_IFREG | 0644
-
-
-class _CentIter(object):
-  def __init__(self, fd):
-    self._fd = fd
-
-  def __iter__(self):
-    return self
-
-  def next(self):
-    fd = self._fd
-
-    type_buf = fd.read(4)
-    type = struct.unpack('<I', type_buf)[0]
-
-    if type != 0x02014b50:  # central directory
-      fd.unread(type_buf)
-      raise StopIteration()
-
-    rec = _CentHeader(fd.read(42))
-    rec.name = fd.read(rec.name_len)
-    fd.read(rec.extra_len)
-    fd.read(rec.comment_len)
-
-    if rec.name.endswith('/'):
-      rec.name = rec.name[:-1]
-      rec.mode = stat.S_IFDIR | 0777
-    return rec
-
-
-class _CentHeader(object):
-  """Information about a single file in the archive.
-     0  version made by                 2 bytes
-     1  version needed to extract       2 bytes
-     2  general purpose bit flag        2 bytes
-     3  compression method              2 bytes
-     4  last mod file time              2 bytes
-     5  last mod file date              2 bytes
-     6  crc-32                          4 bytes
-     7  compressed size                 4 bytes
-     8  uncompressed size               4 bytes
-     9  file name length                2 bytes
-    10  extra field length              2 bytes
-    11  file comment length             2 bytes
-    12  disk number start               2 bytes
-    13  internal file attributes        2 bytes
-    14  external file attributes        4 bytes
-    15  relative offset of local header 4 bytes
-  """
-  def __init__(self, raw_bin):
-    rec = struct.unpack('<6H3I5H2I', raw_bin)
-    self.name_len = rec[9]
-    self.extra_len = rec[10]
-    self.comment_len = rec[11]
-
-    if (rec[0] & 0xff00) == 0x0300:  # UNIX
-      self.mode = rec[14] >> 16
-    else:
-      self.mode = stat.S_IFREG | 0644
-
-
-class _UngetStream(object):
-  """File like object to read and rewind a stream.
-  """
-  def __init__(self, fd):
-    self._fd = fd
-    self._buf = None
-
-  def read(self, size = -1):
-    r = []
-    try:
-      if size >= 0:
-        self._ReadChunk(r, size)
-      else:
-        while True:
-          self._ReadChunk(r, 2048)
-    except EOFError:
-      pass
-
-    if len(r) == 1:
-      return r[0]
-    return ''.join(r)
-
-  def unread(self, buf):
-    b = self._buf
-    if b is None or len(b) == 0:
-      self._buf = buf
-    else:
-      self._buf = buf + b
-
-  def _ReadChunk(self, r, size):
-    b = self._buf
-    try:
-      while size > 0:
-        if b is None or len(b) == 0:
-          b = self._Inflate(self._fd.read(2048))
-          if not b:
-            raise EOFError()
-          continue
-
-        use = min(size, len(b))
-        r.append(b[:use])
-        b = b[use:]
-        size -= use
-    finally:
-      self._buf = b
-
-  def _Inflate(self, b):
-    return b
-
-
-class _FixedLengthStream(_UngetStream):
-  """File like object to read a fixed length stream.
-  """
-  def __init__(self, fd, have):
-    _UngetStream.__init__(self, fd)
-    self._have = have
-
-  def _Inflate(self, b):
-    n = self._have
-    if n == 0:
-      self._fd.unread(b)
-      return None
-
-    if len(b) > n:
-      self._fd.unread(b[n:])
-      b = b[:n]
-    self._have -= len(b)
-    return b
-
-
-class _InflateStream(_UngetStream):
-  """Inflates the stream as it reads input.
-  """
-  def __init__(self, fd):
-    _UngetStream.__init__(self, fd)
-    self._z = zlib.decompressobj(-zlib.MAX_WBITS)
-
-  def _Inflate(self, b):
-    z = self._z
-    if not z:
-      self._fd.unread(b)
-      return None
-
-    b = z.decompress(b)
-    if z.unconsumed_tail != '':
-      self._fd.unread(z.unconsumed_tail)
-    elif z.unused_data != '':
-      self._fd.unread(z.unused_data)
-      self._z = None
-    return b

main.py

@@ -28,6 +28,7 @@ import re
 import sys
 
 from command import InteractiveCommand, PagedCommand
+from editor import Editor
 from error import NoSuchProjectError
 from error import RepoChangedException
 from manifest import Manifest
@@ -77,6 +78,7 @@ class _Repo(object):
 
     cmd.repodir = self.repodir
     cmd.manifest = Manifest(cmd.repodir)
+    Editor.globalConfig = cmd.manifest.globalConfig
 
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig

manifest.py

@@ -17,10 +17,7 @@ import os
 import sys
 import xml.dom.minidom
 
-from editor import Editor
 from git_config import GitConfig, IsId
-from import_tar import ImportTar
-from import_zip import ImportZip
 from project import Project, MetaProject, R_TAGS
 from remote import Remote
 from error import ManifestParseError
@@ -42,9 +39,7 @@ class Manifest(object):
     self.repodir = os.path.abspath(repodir)
     self.topdir = os.path.dirname(self.repodir)
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
-
     self.globalConfig = GitConfig.ForUser()
-    Editor.globalConfig = self.globalConfig
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -248,78 +243,8 @@ class Manifest(object):
       elif n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
 
-    to_resolve = []
-    by_version = {}
-
-    for n in node.childNodes:
-      if n.nodeName == 'import':
-        self._ParseImport(project, n, to_resolve, by_version)
-
-    for pair in to_resolve:
-      sn, pr = pair
-      try:
-        sn.SetParent(by_version[pr].commit)
-      except KeyError:
-        raise ManifestParseError, \
-              'snapshot %s not in project %s in %s' % \
-              (pr, project.name, self.manifestFile)
-
     return project
 
-  def _ParseImport(self, project, import_node, to_resolve, by_version):
-    first_url = None
-    for node in import_node.childNodes:
-      if node.nodeName == 'mirror':
-        first_url = self._reqatt(node, 'url')
-        break
-    if not first_url:
-      raise ManifestParseError, \
-            'mirror url required for project %s in %s' % \
-            (project.name, self.manifestFile)
-
-    imp = None
-    for cls in [ImportTar, ImportZip]:
-      if cls.CanAccept(first_url):
-        imp = cls()
-        break
-    if not imp:
-      raise ManifestParseError, \
-            'snapshot %s unsupported for project %s in %s' % \
-            (first_url, project.name, self.manifestFile)
-
-    imp.SetProject(project)
-
-    for node in import_node.childNodes:
-      if node.nodeName == 'remap':
-        old = node.getAttribute('strip')
-        new = node.getAttribute('insert')
-        imp.RemapPath(old, new)
-
-      elif node.nodeName == 'mirror':
-        imp.AddUrl(self._reqatt(node, 'url'))
-
-    for node in import_node.childNodes:
-      if node.nodeName == 'snapshot':
-        sn = imp.Clone()
-        sn.SetVersion(self._reqatt(node, 'version'))
-        sn.SetCommit(node.getAttribute('check'))
-
-        pr = node.getAttribute('prior')
-        if pr:
-          if IsId(pr):
-            sn.SetParent(pr)
-          else:
-            to_resolve.append((sn, pr))
-
-        rev = R_TAGS + sn.TagName
-
-        if rev in project.snapshots:
-          raise ManifestParseError, \
-                'duplicate snapshot %s for project %s in %s' % \
-                (sn.version, project.name, self.manifestFile)
-        project.snapshots[rev] = sn
-        by_version[sn.version] = sn
-
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
     dest = self._reqatt(node, 'dest')

project.py

@@ -466,8 +466,6 @@ class Project(object):
     for r in self.extraRemotes.values():
       if not self._RemoteFetch(r.name):
         return False
-    if not self._SnapshotDownload():
-      return False
     if not self._RemoteFetch():
       return False
     self._RepairAndroidImportErrors()
@@ -629,33 +627,6 @@ class Project(object):
     self._CopyFiles()
     return True
 
-  def _SnapshotDownload(self):
-    if self.snapshots:
-      have = set(self._allrefs.keys())
-      need = []
-
-      for tag, sn in self.snapshots.iteritems():
-        if tag not in have:
-          need.append(sn)
-
-      if need:
-        print >>sys.stderr, """
-  ***   Downloading source(s) from a mirror site.     ***
-  ***   If the network hangs, kill and restart repo.  ***
-"""
-        for sn in need:
-          try:
-            sn.Import()
-          except ImportError, e:
-            print >>sys.stderr, \
-              'error: Cannot import %s: %s' \
-              % (self.name, e)
-            return False
-        cmd = ['repack', '-a', '-d', '-f', '-l']
-        if GitCommand(self, cmd, bare = True).Wait() != 0:
-          return False
-    return True
-
   def AddCopyFile(self, src, dest):
     # dest should already be an absolute path, but src is project relative
     # make src an absolute path
@@ -762,41 +733,9 @@ class Project(object):
   def _RemoteFetch(self, name=None):
     if not name:
       name = self.remote.name
-
-    hide_errors = False
-    if self.extraRemotes or self.snapshots:
-      hide_errors = True
-
-    proc = GitCommand(self,
+    return GitCommand(self,
                       ['fetch', name],
-                      bare = True,
-                      capture_stderr = hide_errors)
-    if hide_errors:
-      err = proc.process.stderr.fileno()
-      buf = ''
-      while True:
-        b = os.read(err, 256)
-        if b:
-          buf += b
-        while buf:
-          r = buf.find('remote: error: unable to find ')
-          if r >= 0:
-            lf = buf.find('\n')
-            if lf < 0:
-              break
-            buf = buf[lf + 1:]
-            continue
-
-          cr = buf.find('\r')
-          if cr < 0:
-            break
-          os.write(2, buf[0:cr + 1])
-          buf = buf[cr + 1:]
-        if not b:
-          if buf:
-            os.write(2, buf)
-          break
-    return proc.Wait() == 0
+                      bare = True).Wait() == 0
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -836,7 +775,11 @@ class Project(object):
       self.config.SetString('core.bare', None)
 
       hooks = self._gitdir_path('hooks')
-      for old_hook in os.listdir(hooks):
+      try:
+        to_rm = os.listdir(hooks)
+      except OSError:
+        to_rm = []
+      for old_hook in to_rm:
         os.remove(os.path.join(hooks, old_hook))
 
       # TODO(sop) install custom repo hooks

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 5)
+VERSION = (1, 6)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -202,6 +202,7 @@ def _CheckGitVersion():
   proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   ver_str = proc.stdout.read().strip()
   proc.stdout.close()
+  proc.wait()
 
   if not ver_str.startswith('git version '):
     print >>sys.stderr, 'error: "%s" unsupported' % ver_str

subcmds/compute_snapshot_check.py

@@ -1,169 +0,0 @@
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-import tempfile
-
-from command import Command
-from error import GitError, NoSuchProjectError
-from git_config import IsId
-from import_tar import ImportTar
-from import_zip import ImportZip
-from project import Project
-from remote import Remote
-
-def _ToCommit(project, rev):
-  return project.bare_git.rev_parse('--verify', '%s^0' % rev)
-
-def _Missing(project, rev):
-  return project._revlist('--objects', rev, '--not', '--all')
-
-
-class ComputeSnapshotCheck(Command):
-  common = False
-  helpSummary = "Compute the check value for a new snapshot"
-  helpUsage = """
-%prog -p NAME -v VERSION -s FILE [options]
-"""
-  helpDescription = """
-%prog computes and then displays the proper check value for a
-snapshot, so it can be pasted into the manifest file for a project.
-"""
-
-  def _Options(self, p):
-    g = p.add_option_group('Snapshot description options')
-    g.add_option('-p', '--project',
-                 dest='project', metavar='NAME',
-                 help='destination project name')
-    g.add_option('-v', '--version',
-                 dest='version', metavar='VERSION',
-                 help='upstream version/revision identifier')
-    g.add_option('-s', '--snapshot',
-                 dest='snapshot', metavar='PATH',
-                 help='local tarball path')
-    g.add_option('--new-project',
-                 dest='new_project', action='store_true',
-                 help='destinition is a new project')
-    g.add_option('--keep',
-                 dest='keep_git', action='store_true',
-                 help='keep the temporary git repository')
-
-    g = p.add_option_group('Base revision grafting options')
-    g.add_option('--prior',
-                 dest='prior', metavar='COMMIT',
-                 help='prior revision checksum')
-
-    g = p.add_option_group('Path mangling options')
-    g.add_option('--strip-prefix',
-                 dest='strip_prefix', metavar='PREFIX',
-                 help='remove prefix from all paths on import')
-    g.add_option('--insert-prefix',
-                 dest='insert_prefix', metavar='PREFIX',
-                 help='insert prefix before all paths on import')
-
-
-  def _Compute(self, opt):
-    try:
-      real_project = self.GetProjects([opt.project])[0]
-    except NoSuchProjectError:
-      if opt.new_project:
-        print >>sys.stderr, \
-          "warning: project '%s' does not exist" % opt.project
-      else:
-        raise NoSuchProjectError(opt.project)
-
-    self._tmpdir = tempfile.mkdtemp()
-    project = Project(manifest = self.manifest,
-                      name = opt.project,
-                      remote = Remote('origin'),
-                      gitdir = os.path.join(self._tmpdir, '.git'),
-                      worktree = self._tmpdir,
-                      relpath = opt.project,
-                      revision = 'refs/heads/master')
-    project._InitGitDir()
-
-    url = 'file://%s' % os.path.abspath(opt.snapshot)
-
-    imp = None
-    for cls in [ImportTar, ImportZip]:
-      if cls.CanAccept(url):
-        imp = cls()
-        break
-    if not imp:
-      print >>sys.stderr, 'error: %s unsupported' % opt.snapshot
-      sys.exit(1)
-
-    imp.SetProject(project)
-    imp.SetVersion(opt.version)
-    imp.AddUrl(url)
-
-    if opt.prior:
-      if opt.new_project:
-        if not IsId(opt.prior):
-          print >>sys.stderr, 'error: --prior=%s not valid' % opt.prior
-          sys.exit(1)
-      else:
-        try:
-          opt.prior = _ToCommit(real_project, opt.prior)
-          missing = _Missing(real_project, opt.prior)
-        except GitError, e:
-          print >>sys.stderr,\
-            'error: --prior=%s not valid\n%s' \
-            % (opt.prior, e)
-          sys.exit(1)
-        if missing:
-          print >>sys.stderr,\
-            'error: --prior=%s is valid, but is not reachable' \
-            % opt.prior
-          sys.exit(1)
-      imp.SetParent(opt.prior)
-
-    src = opt.strip_prefix
-    dst = opt.insert_prefix
-    if src or dst:
-      if src is None:
-        src = ''
-      if dst is None:
-        dst = ''
-      imp.RemapPath(src, dst)
-    commitId = imp.Import()
-
-    print >>sys.stderr,"%s\t%s" % (commitId, imp.version)
-    return project
-
-  def Execute(self, opt, args):
-    if args \
-       or not opt.project \
-       or not opt.version \
-       or not opt.snapshot:
-      self.Usage()
-
-    success = False
-    project = None
-    try:
-      self._tmpdir = None
-      project = self._Compute(opt)
-    finally:
-      if project and opt.keep_git:
-        print 'GIT_DIR = %s' % (project.gitdir)
-      elif self._tmpdir:
-        for root, dirs, files in os.walk(self._tmpdir, topdown=False):
-          for name in files:
-            os.remove(os.path.join(root, name))
-          for name in dirs:
-            os.rmdir(os.path.join(root, name))
-        os.rmdir(self._tmpdir)
-