mirror of
https://gerrit.googlesource.com/git-repo
synced 2024-12-21 07:16:21 +00:00
Remove import_tar, import_zip and the <snapshot> elements
Now that repo relies only on the git data stream (as it is much faster to download through) we don't really need to be parsing the <snapshot> elements within manifest. Its a lot of complex code to convert the tar (or zip) through to a fast import stream, and we just aren't calling it anymore. Signed-off-by: Shawn O. Pearce <sop@google.com>
This commit is contained in:
Shawn O. Pearce
parent
90be5c0839
commit
df830f1238
422
import_ext.py
422
import_ext.py
@ -1,422 +0,0 @@
|
||||
#
|
||||
# Copyright (C) 2008 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import random
|
||||
import stat
|
||||
import sys
|
||||
import urllib2
|
||||
import StringIO
|
||||
|
||||
from error import GitError, ImportError
|
||||
from git_command import GitCommand
|
||||
|
||||
class ImportExternal(object):
|
||||
"""Imports a single revision from a non-git data source.
|
||||
Suitable for use to import a tar or zip based snapshot.
|
||||
"""
|
||||
def __init__(self):
|
||||
self._marks = 0
|
||||
self._files = {}
|
||||
self._tempref = 'refs/repo-external/import'
|
||||
|
||||
self._urls = []
|
||||
self._remap = []
|
||||
self.parent = None
|
||||
self._user_name = 'Upstream'
|
||||
self._user_email = 'upstream-import@none'
|
||||
self._user_when = 1000000
|
||||
|
||||
self.commit = None
|
||||
|
||||
def Clone(self):
|
||||
r = self.__class__()
|
||||
|
||||
r.project = self.project
|
||||
for u in self._urls:
|
||||
r._urls.append(u)
|
||||
for p in self._remap:
|
||||
r._remap.append(_PathMap(r, p._old, p._new))
|
||||
|
||||
return r
|
||||
|
||||
def SetProject(self, project):
|
||||
self.project = project
|
||||
|
||||
def SetVersion(self, version):
|
||||
self.version = version
|
||||
|
||||
def AddUrl(self, url):
|
||||
self._urls.append(url)
|
||||
|
||||
def SetParent(self, commit_hash):
|
||||
self.parent = commit_hash
|
||||
|
||||
def SetCommit(self, commit_hash):
|
||||
self.commit = commit_hash
|
||||
|
||||
def RemapPath(self, old, new, replace_version=True):
|
||||
self._remap.append(_PathMap(self, old, new))
|
||||
|
||||
@property
|
||||
def TagName(self):
|
||||
v = ''
|
||||
for c in self.version:
|
||||
if c >= '0' and c <= '9':
|
||||
v += c
|
||||
elif c >= 'A' and c <= 'Z':
|
||||
v += c
|
||||
elif c >= 'a' and c <= 'z':
|
||||
v += c
|
||||
elif c in ('-', '_', '.', '/', '+', '@'):
|
||||
v += c
|
||||
return 'upstream/%s' % v
|
||||
|
||||
@property
|
||||
def PackageName(self):
|
||||
n = self.project.name
|
||||
if n.startswith('platform/'):
|
||||
# This was not my finest moment...
|
||||
#
|
||||
n = n[len('platform/'):]
|
||||
return n
|
||||
|
||||
def Import(self):
|
||||
self._need_graft = False
|
||||
if self.parent:
|
||||
try:
|
||||
self.project.bare_git.cat_file('-e', self.parent)
|
||||
except GitError:
|
||||
self._need_graft = True
|
||||
|
||||
gfi = GitCommand(self.project,
|
||||
['fast-import', '--force', '--quiet'],
|
||||
bare = True,
|
||||
provide_stdin = True)
|
||||
try:
|
||||
self._out = gfi.stdin
|
||||
|
||||
try:
|
||||
self._UnpackFiles()
|
||||
self._MakeCommit()
|
||||
self._out.flush()
|
||||
finally:
|
||||
rc = gfi.Wait()
|
||||
if rc != 0:
|
||||
raise ImportError('fast-import failed')
|
||||
|
||||
if self._need_graft:
|
||||
id = self._GraftCommit()
|
||||
else:
|
||||
id = self.project.bare_git.rev_parse('%s^0' % self._tempref)
|
||||
|
||||
if self.commit and self.commit != id:
|
||||
raise ImportError('checksum mismatch: %s expected,'
|
||||
' %s imported' % (self.commit, id))
|
||||
|
||||
self._MakeTag(id)
|
||||
return id
|
||||
finally:
|
||||
try:
|
||||
self.project.bare_git.DeleteRef(self._tempref)
|
||||
except GitError:
|
||||
pass
|
||||
|
||||
def _PickUrl(self, failed):
|
||||
u = map(lambda x: x.replace('%version%', self.version), self._urls)
|
||||
for f in failed:
|
||||
if f in u:
|
||||
u.remove(f)
|
||||
if len(u) == 0:
|
||||
return None
|
||||
return random.choice(u)
|
||||
|
||||
def _OpenUrl(self):
|
||||
failed = {}
|
||||
while True:
|
||||
url = self._PickUrl(failed.keys())
|
||||
if url is None:
|
||||
why = 'Cannot download %s' % self.project.name
|
||||
|
||||
if failed:
|
||||
why += ': one or more mirrors are down\n'
|
||||
bad_urls = list(failed.keys())
|
||||
bad_urls.sort()
|
||||
for url in bad_urls:
|
||||
why += ' %s: %s\n' % (url, failed[url])
|
||||
else:
|
||||
why += ': no mirror URLs'
|
||||
raise ImportError(why)
|
||||
|
||||
print >>sys.stderr, "Getting %s ..." % url
|
||||
try:
|
||||
return urllib2.urlopen(url), url
|
||||
except urllib2.HTTPError, e:
|
||||
failed[url] = e.code
|
||||
except urllib2.URLError, e:
|
||||
failed[url] = e.reason[1]
|
||||
except OSError, e:
|
||||
failed[url] = e.strerror
|
||||
|
||||
def _UnpackFiles(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _NextMark(self):
|
||||
self._marks += 1
|
||||
return self._marks
|
||||
|
||||
def _UnpackOneFile(self, mode, size, name, fd):
|
||||
if stat.S_ISDIR(mode): # directory
|
||||
return
|
||||
else:
|
||||
mode = self._CleanMode(mode, name)
|
||||
|
||||
old_name = name
|
||||
name = self._CleanName(name)
|
||||
|
||||
if stat.S_ISLNK(mode) and self._remap:
|
||||
# The link is relative to the old_name, and may need to
|
||||
# be rewritten according to our remap rules if it goes
|
||||
# up high enough in the tree structure.
|
||||
#
|
||||
dest = self._RewriteLink(fd.read(size), old_name, name)
|
||||
fd = StringIO.StringIO(dest)
|
||||
size = len(dest)
|
||||
|
||||
fi = _File(mode, name, self._NextMark())
|
||||
|
||||
self._out.write('blob\n')
|
||||
self._out.write('mark :%d\n' % fi.mark)
|
||||
self._out.write('data %d\n' % size)
|
||||
while size > 0:
|
||||
n = min(2048, size)
|
||||
self._out.write(fd.read(n))
|
||||
size -= n
|
||||
self._out.write('\n')
|
||||
self._files[fi.name] = fi
|
||||
|
||||
def _SetFileMode(self, name, mode):
|
||||
if not stat.S_ISDIR(mode):
|
||||
mode = self._CleanMode(mode, name)
|
||||
name = self._CleanName(name)
|
||||
try:
|
||||
fi = self._files[name]
|
||||
except KeyError:
|
||||
raise ImportError('file %s was not unpacked' % name)
|
||||
fi.mode = mode
|
||||
|
||||
def _RewriteLink(self, dest, relto_old, relto_new):
|
||||
# Drop the last components of the symlink itself
|
||||
# as the dest is relative to the directory its in.
|
||||
#
|
||||
relto_old = _TrimPath(relto_old)
|
||||
relto_new = _TrimPath(relto_new)
|
||||
|
||||
# Resolve the link to be absolute from the top of
|
||||
# the archive, so we can remap its destination.
|
||||
#
|
||||
while dest.find('/./') >= 0 or dest.find('//') >= 0:
|
||||
dest = dest.replace('/./', '/')
|
||||
dest = dest.replace('//', '/')
|
||||
|
||||
if dest.startswith('../') or dest.find('/../') > 0:
|
||||
dest = _FoldPath('%s/%s' % (relto_old, dest))
|
||||
|
||||
for pm in self._remap:
|
||||
if pm.Matches(dest):
|
||||
dest = pm.Apply(dest)
|
||||
break
|
||||
|
||||
dest, relto_new = _StripCommonPrefix(dest, relto_new)
|
||||
while relto_new:
|
||||
i = relto_new.find('/')
|
||||
if i > 0:
|
||||
relto_new = relto_new[i + 1:]
|
||||
else:
|
||||
relto_new = ''
|
||||
dest = '../' + dest
|
||||
return dest
|
||||
|
||||
def _CleanMode(self, mode, name):
|
||||
if stat.S_ISREG(mode): # regular file
|
||||
if (mode & 0111) == 0:
|
||||
return 0644
|
||||
else:
|
||||
return 0755
|
||||
elif stat.S_ISLNK(mode): # symlink
|
||||
return stat.S_IFLNK
|
||||
else:
|
||||
raise ImportError('invalid mode %o in %s' % (mode, name))
|
||||
|
||||
def _CleanName(self, name):
|
||||
old_name = name
|
||||
for pm in self._remap:
|
||||
if pm.Matches(name):
|
||||
name = pm.Apply(name)
|
||||
break
|
||||
while name.startswith('/'):
|
||||
name = name[1:]
|
||||
if not name:
|
||||
raise ImportError('path %s is empty after remap' % old_name)
|
||||
if name.find('/./') >= 0 or name.find('/../') >= 0:
|
||||
raise ImportError('path %s contains relative parts' % name)
|
||||
return name
|
||||
|
||||
def _MakeCommit(self):
|
||||
msg = '%s %s\n' % (self.PackageName, self.version)
|
||||
|
||||
self._out.write('commit %s\n' % self._tempref)
|
||||
self._out.write('committer %s <%s> %d +0000\n' % (
|
||||
self._user_name,
|
||||
self._user_email,
|
||||
self._user_when))
|
||||
self._out.write('data %d\n' % len(msg))
|
||||
self._out.write(msg)
|
||||
self._out.write('\n')
|
||||
if self.parent and not self._need_graft:
|
||||
self._out.write('from %s^0\n' % self.parent)
|
||||
self._out.write('deleteall\n')
|
||||
|
||||
for f in self._files.values():
|
||||
self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name))
|
||||
self._out.write('\n')
|
||||
|
||||
def _GraftCommit(self):
|
||||
raw = self.project.bare_git.cat_file('commit', self._tempref)
|
||||
raw = raw.split("\n")
|
||||
while raw[1].startswith('parent '):
|
||||
del raw[1]
|
||||
raw.insert(1, 'parent %s' % self.parent)
|
||||
id = self._WriteObject('commit', "\n".join(raw))
|
||||
|
||||
graft_file = os.path.join(self.project.gitdir, 'info/grafts')
|
||||
if os.path.exists(graft_file):
|
||||
graft_list = open(graft_file, 'rb').read().split("\n")
|
||||
if graft_list and graft_list[-1] == '':
|
||||
del graft_list[-1]
|
||||
else:
|
||||
graft_list = []
|
||||
|
||||
exists = False
|
||||
for line in graft_list:
|
||||
if line == id:
|
||||
exists = True
|
||||
break
|
||||
|
||||
if not exists:
|
||||
graft_list.append(id)
|
||||
graft_list.append('')
|
||||
fd = open(graft_file, 'wb')
|
||||
fd.write("\n".join(graft_list))
|
||||
fd.close()
|
||||
|
||||
return id
|
||||
|
||||
def _MakeTag(self, id):
|
||||
name = self.TagName
|
||||
|
||||
raw = []
|
||||
raw.append('object %s' % id)
|
||||
raw.append('type commit')
|
||||
raw.append('tag %s' % name)
|
||||
raw.append('tagger %s <%s> %d +0000' % (
|
||||
self._user_name,
|
||||
self._user_email,
|
||||
self._user_when))
|
||||
raw.append('')
|
||||
raw.append('%s %s\n' % (self.PackageName, self.version))
|
||||
|
||||
tagid = self._WriteObject('tag', "\n".join(raw))
|
||||
self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid)
|
||||
|
||||
def _WriteObject(self, type, data):
|
||||
wo = GitCommand(self.project,
|
||||
['hash-object', '-t', type, '-w', '--stdin'],
|
||||
bare = True,
|
||||
provide_stdin = True,
|
||||
capture_stdout = True,
|
||||
capture_stderr = True)
|
||||
wo.stdin.write(data)
|
||||
if wo.Wait() != 0:
|
||||
raise GitError('cannot create %s from (%s)' % (type, data))
|
||||
return wo.stdout[:-1]
|
||||
|
||||
|
||||
def _TrimPath(path):
|
||||
i = path.rfind('/')
|
||||
if i > 0:
|
||||
path = path[0:i]
|
||||
return ''
|
||||
|
||||
def _StripCommonPrefix(a, b):
|
||||
while True:
|
||||
ai = a.find('/')
|
||||
bi = b.find('/')
|
||||
if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]:
|
||||
a = a[ai + 1:]
|
||||
b = b[bi + 1:]
|
||||
else:
|
||||
break
|
||||
return a, b
|
||||
|
||||
def _FoldPath(path):
|
||||
while True:
|
||||
if path.startswith('../'):
|
||||
return path
|
||||
|
||||
i = path.find('/../')
|
||||
if i <= 0:
|
||||
if path.startswith('/'):
|
||||
return path[1:]
|
||||
return path
|
||||
|
||||
lhs = path[0:i]
|
||||
rhs = path[i + 4:]
|
||||
|
||||
i = lhs.rfind('/')
|
||||
if i > 0:
|
||||
path = lhs[0:i + 1] + rhs
|
||||
else:
|
||||
path = rhs
|
||||
|
||||
class _File(object):
|
||||
def __init__(self, mode, name, mark):
|
||||
self.mode = mode
|
||||
self.name = name
|
||||
self.mark = mark
|
||||
|
||||
|
||||
class _PathMap(object):
|
||||
def __init__(self, imp, old, new):
|
||||
self._imp = imp
|
||||
self._old = old
|
||||
self._new = new
|
||||
|
||||
def _r(self, p):
|
||||
return p.replace('%version%', self._imp.version)
|
||||
|
||||
@property
|
||||
def old(self):
|
||||
return self._r(self._old)
|
||||
|
||||
@property
|
||||
def new(self):
|
||||
return self._r(self._new)
|
||||
|
||||
def Matches(self, name):
|
||||
return name.startswith(self.old)
|
||||
|
||||
def Apply(self, name):
|
||||
return self.new + name[len(self.old):]
|
||||
206
import_tar.py
206
import_tar.py
@ -1,206 +0,0 @@
|
||||
#
|
||||
# Copyright (C) 2008 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import bz2
|
||||
import stat
|
||||
import tarfile
|
||||
import zlib
|
||||
import StringIO
|
||||
|
||||
from import_ext import ImportExternal
|
||||
from error import ImportError
|
||||
|
||||
class ImportTar(ImportExternal):
|
||||
"""Streams a (optionally compressed) tar file from the network
|
||||
directly into a Project's Git repository.
|
||||
"""
|
||||
@classmethod
|
||||
def CanAccept(cls, url):
|
||||
"""Can this importer read and unpack the data stored at url?
|
||||
"""
|
||||
if url.endswith('.tar.gz') or url.endswith('.tgz'):
|
||||
return True
|
||||
if url.endswith('.tar.bz2'):
|
||||
return True
|
||||
if url.endswith('.tar'):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _UnpackFiles(self):
|
||||
url_fd, url = self._OpenUrl()
|
||||
try:
|
||||
if url.endswith('.tar.gz') or url.endswith('.tgz'):
|
||||
tar_fd = _Gzip(url_fd)
|
||||
elif url.endswith('.tar.bz2'):
|
||||
tar_fd = _Bzip2(url_fd)
|
||||
elif url.endswith('.tar'):
|
||||
tar_fd = _Raw(url_fd)
|
||||
else:
|
||||
raise ImportError('non-tar file extension: %s' % url)
|
||||
|
||||
try:
|
||||
tar = tarfile.TarFile(name = url,
|
||||
mode = 'r',
|
||||
fileobj = tar_fd)
|
||||
try:
|
||||
for entry in tar:
|
||||
mode = entry.mode
|
||||
|
||||
if (mode & 0170000) == 0:
|
||||
if entry.isdir():
|
||||
mode |= stat.S_IFDIR
|
||||
elif entry.isfile() or entry.islnk(): # hard links as files
|
||||
mode |= stat.S_IFREG
|
||||
elif entry.issym():
|
||||
mode |= stat.S_IFLNK
|
||||
|
||||
if stat.S_ISLNK(mode): # symlink
|
||||
data_fd = StringIO.StringIO(entry.linkname)
|
||||
data_sz = len(entry.linkname)
|
||||
elif stat.S_ISDIR(mode): # directory
|
||||
data_fd = StringIO.StringIO('')
|
||||
data_sz = 0
|
||||
else:
|
||||
data_fd = tar.extractfile(entry)
|
||||
data_sz = entry.size
|
||||
|
||||
self._UnpackOneFile(mode, data_sz, entry.name, data_fd)
|
||||
finally:
|
||||
tar.close()
|
||||
finally:
|
||||
tar_fd.close()
|
||||
finally:
|
||||
url_fd.close()
|
||||
|
||||
|
||||
|
||||
class _DecompressStream(object):
|
||||
"""file like object to decompress a tar stream
|
||||
"""
|
||||
def __init__(self, fd):
|
||||
self._fd = fd
|
||||
self._pos = 0
|
||||
self._buf = None
|
||||
|
||||
def tell(self):
|
||||
return self._pos
|
||||
|
||||
def seek(self, offset):
|
||||
d = offset - self._pos
|
||||
if d > 0:
|
||||
self.read(d)
|
||||
elif d == 0:
|
||||
pass
|
||||
else:
|
||||
raise NotImplementedError, 'seek backwards'
|
||||
|
||||
def close(self):
|
||||
self._fd = None
|
||||
|
||||
def read(self, size = -1):
|
||||
if not self._fd:
|
||||
raise EOFError, 'Reached EOF'
|
||||
|
||||
r = []
|
||||
try:
|
||||
if size >= 0:
|
||||
self._ReadChunk(r, size)
|
||||
else:
|
||||
while True:
|
||||
self._ReadChunk(r, 2048)
|
||||
except EOFError:
|
||||
pass
|
||||
|
||||
if len(r) == 1:
|
||||
r = r[0]
|
||||
else:
|
||||
r = ''.join(r)
|
||||
self._pos += len(r)
|
||||
return r
|
||||
|
||||
def _ReadChunk(self, r, size):
|
||||
b = self._buf
|
||||
try:
|
||||
while size > 0:
|
||||
if b is None or len(b) == 0:
|
||||
b = self._Decompress(self._fd.read(2048))
|
||||
continue
|
||||
|
||||
use = min(size, len(b))
|
||||
r.append(b[:use])
|
||||
b = b[use:]
|
||||
size -= use
|
||||
finally:
|
||||
self._buf = b
|
||||
|
||||
def _Decompress(self, b):
|
||||
raise NotImplementedError, '_Decompress'
|
||||
|
||||
|
||||
class _Raw(_DecompressStream):
|
||||
"""file like object for an uncompressed stream
|
||||
"""
|
||||
def __init__(self, fd):
|
||||
_DecompressStream.__init__(self, fd)
|
||||
|
||||
def _Decompress(self, b):
|
||||
return b
|
||||
|
||||
|
||||
class _Bzip2(_DecompressStream):
|
||||
"""file like object to decompress a .bz2 stream
|
||||
"""
|
||||
def __init__(self, fd):
|
||||
_DecompressStream.__init__(self, fd)
|
||||
self._bz = bz2.BZ2Decompressor()
|
||||
|
||||
def _Decompress(self, b):
|
||||
return self._bz.decompress(b)
|
||||
|
||||
|
||||
_FHCRC, _FEXTRA, _FNAME, _FCOMMENT = 2, 4, 8, 16
|
||||
class _Gzip(_DecompressStream):
|
||||
"""file like object to decompress a .gz stream
|
||||
"""
|
||||
def __init__(self, fd):
|
||||
_DecompressStream.__init__(self, fd)
|
||||
self._z = zlib.decompressobj(-zlib.MAX_WBITS)
|
||||
|
||||
magic = fd.read(2)
|
||||
if magic != '\037\213':
|
||||
raise IOError, 'Not a gzipped file'
|
||||
|
||||
method = ord(fd.read(1))
|
||||
if method != 8:
|
||||
raise IOError, 'Unknown compression method'
|
||||
|
||||
flag = ord(fd.read(1))
|
||||
fd.read(6)
|
||||
|
||||
if flag & _FEXTRA:
|
||||
xlen = ord(fd.read(1))
|
||||
xlen += 256 * ord(fd.read(1))
|
||||
fd.read(xlen)
|
||||
if flag & _FNAME:
|
||||
while fd.read(1) != '\0':
|
||||
pass
|
||||
if flag & _FCOMMENT:
|
||||
while fd.read(1) != '\0':
|
||||
pass
|
||||
if flag & _FHCRC:
|
||||
fd.read(2)
|
||||
|
||||
def _Decompress(self, b):
|
||||
return self._z.decompress(b)
|
||||
345
import_zip.py
345
import_zip.py
@ -1,345 +0,0 @@
|
||||
#
|
||||
# Copyright (C) 2008 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import stat
|
||||
import struct
|
||||
import zlib
|
||||
import cStringIO
|
||||
|
||||
from import_ext import ImportExternal
|
||||
from error import ImportError
|
||||
|
||||
class ImportZip(ImportExternal):
|
||||
"""Streams a zip file from the network directly into a Project's
|
||||
Git repository.
|
||||
"""
|
||||
@classmethod
|
||||
def CanAccept(cls, url):
|
||||
"""Can this importer read and unpack the data stored at url?
|
||||
"""
|
||||
if url.endswith('.zip') or url.endswith('.jar'):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _UnpackFiles(self):
|
||||
url_fd, url = self._OpenUrl()
|
||||
try:
|
||||
if not self.__class__.CanAccept(url):
|
||||
raise ImportError('non-zip file extension: %s' % url)
|
||||
|
||||
zip = _ZipFile(url_fd)
|
||||
for entry in zip.FileRecords():
|
||||
data = zip.Open(entry).read()
|
||||
sz = len(data)
|
||||
|
||||
if data and _SafeCRLF(data):
|
||||
data = data.replace('\r\n', '\n')
|
||||
sz = len(data)
|
||||
|
||||
fd = cStringIO.StringIO(data)
|
||||
self._UnpackOneFile(entry.mode, sz, entry.name, fd)
|
||||
zip.Close(entry)
|
||||
|
||||
for entry in zip.CentralDirectory():
|
||||
self._SetFileMode(entry.name, entry.mode)
|
||||
|
||||
zip.CheckTail()
|
||||
finally:
|
||||
url_fd.close()
|
||||
|
||||
|
||||
def _SafeCRLF(data):
|
||||
"""Is it reasonably safe to perform a CRLF->LF conversion?
|
||||
|
||||
If the stream contains a NUL byte it is likely binary,
|
||||
and thus a CRLF->LF conversion may damage the stream.
|
||||
|
||||
If the only NUL is in the last position of the stream,
|
||||
but it otherwise can do a CRLF<->LF conversion we do
|
||||
the CRLF conversion anyway. At least one source ZIP
|
||||
file has this structure in its source code.
|
||||
|
||||
If every occurrance of a CR and LF is paired up as a
|
||||
CRLF pair then the conversion is safely bi-directional.
|
||||
s/\r\n/\n/g == s/\n/\r\\n/g can convert between them.
|
||||
"""
|
||||
nul = data.find('\0')
|
||||
if 0 <= nul and nul < (len(data) - 1):
|
||||
return False
|
||||
|
||||
n_lf = 0
|
||||
last = 0
|
||||
while True:
|
||||
lf = data.find('\n', last)
|
||||
if lf < 0:
|
||||
break
|
||||
if lf == 0 or data[lf - 1] != '\r':
|
||||
return False
|
||||
last = lf + 1
|
||||
n_lf += 1
|
||||
return n_lf > 0
|
||||
|
||||
class _ZipFile(object):
|
||||
"""Streaming iterator to parse a zip file on the fly.
|
||||
"""
|
||||
def __init__(self, fd):
|
||||
self._fd = _UngetStream(fd)
|
||||
|
||||
def FileRecords(self):
|
||||
return _FileIter(self._fd)
|
||||
|
||||
def CentralDirectory(self):
|
||||
return _CentIter(self._fd)
|
||||
|
||||
def CheckTail(self):
|
||||
type_buf = self._fd.read(4)
|
||||
type = struct.unpack('<I', type_buf)[0]
|
||||
if type != 0x06054b50: # end of central directory
|
||||
raise ImportError('zip record %x unsupported' % type)
|
||||
|
||||
def Open(self, entry):
|
||||
if entry.is_compressed:
|
||||
return _InflateStream(self._fd)
|
||||
else:
|
||||
if entry.has_trailer:
|
||||
raise ImportError('unable to extract streamed zip')
|
||||
return _FixedLengthStream(self._fd, entry.uncompressed_size)
|
||||
|
||||
def Close(self, entry):
|
||||
if entry.has_trailer:
|
||||
type = struct.unpack('<I', self._fd.read(4))[0]
|
||||
if type == 0x08074b50:
|
||||
# Not a formal type marker, but commonly seen in zips
|
||||
# as the data descriptor signature.
|
||||
#
|
||||
struct.unpack('<3I', self._fd.read(12))
|
||||
else:
|
||||
# No signature for the data descriptor, so read the
|
||||
# remaining fields out of the stream
|
||||
#
|
||||
self._fd.read(8)
|
||||
|
||||
|
||||
class _FileIter(object):
|
||||
def __init__(self, fd):
|
||||
self._fd = fd
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
fd = self._fd
|
||||
|
||||
type_buf = fd.read(4)
|
||||
type = struct.unpack('<I', type_buf)[0]
|
||||
|
||||
if type != 0x04034b50: # local file header
|
||||
fd.unread(type_buf)
|
||||
raise StopIteration()
|
||||
|
||||
rec = _FileHeader(fd.read(26))
|
||||
rec.name = fd.read(rec.name_len)
|
||||
fd.read(rec.extra_len)
|
||||
|
||||
if rec.name.endswith('/'):
|
||||
rec.name = rec.name[:-1]
|
||||
rec.mode = stat.S_IFDIR | 0777
|
||||
return rec
|
||||
|
||||
|
||||
class _FileHeader(object):
|
||||
"""Information about a single file in the archive.
|
||||
0 version needed to extract 2 bytes
|
||||
1 general purpose bit flag 2 bytes
|
||||
2 compression method 2 bytes
|
||||
3 last mod file time 2 bytes
|
||||
4 last mod file date 2 bytes
|
||||
5 crc-32 4 bytes
|
||||
6 compressed size 4 bytes
|
||||
7 uncompressed size 4 bytes
|
||||
8 file name length 2 bytes
|
||||
9 extra field length 2 bytes
|
||||
"""
|
||||
def __init__(self, raw_bin):
|
||||
rec = struct.unpack('<5H3I2H', raw_bin)
|
||||
|
||||
if rec[2] == 8:
|
||||
self.is_compressed = True
|
||||
elif rec[2] == 0:
|
||||
self.is_compressed = False
|
||||
else:
|
||||
raise ImportError('unrecognized compression format')
|
||||
|
||||
if rec[1] & (1 << 3):
|
||||
self.has_trailer = True
|
||||
else:
|
||||
self.has_trailer = False
|
||||
|
||||
self.compressed_size = rec[6]
|
||||
self.uncompressed_size = rec[7]
|
||||
self.name_len = rec[8]
|
||||
self.extra_len = rec[9]
|
||||
self.mode = stat.S_IFREG | 0644
|
||||
|
||||
|
||||
class _CentIter(object):
|
||||
def __init__(self, fd):
|
||||
self._fd = fd
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
fd = self._fd
|
||||
|
||||
type_buf = fd.read(4)
|
||||
type = struct.unpack('<I', type_buf)[0]
|
||||
|
||||
if type != 0x02014b50: # central directory
|
||||
fd.unread(type_buf)
|
||||
raise StopIteration()
|
||||
|
||||
rec = _CentHeader(fd.read(42))
|
||||
rec.name = fd.read(rec.name_len)
|
||||
fd.read(rec.extra_len)
|
||||
fd.read(rec.comment_len)
|
||||
|
||||
if rec.name.endswith('/'):
|
||||
rec.name = rec.name[:-1]
|
||||
rec.mode = stat.S_IFDIR | 0777
|
||||
return rec
|
||||
|
||||
|
||||
class _CentHeader(object):
|
||||
"""Information about a single file in the archive.
|
||||
0 version made by 2 bytes
|
||||
1 version needed to extract 2 bytes
|
||||
2 general purpose bit flag 2 bytes
|
||||
3 compression method 2 bytes
|
||||
4 last mod file time 2 bytes
|
||||
5 last mod file date 2 bytes
|
||||
6 crc-32 4 bytes
|
||||
7 compressed size 4 bytes
|
||||
8 uncompressed size 4 bytes
|
||||
9 file name length 2 bytes
|
||||
10 extra field length 2 bytes
|
||||
11 file comment length 2 bytes
|
||||
12 disk number start 2 bytes
|
||||
13 internal file attributes 2 bytes
|
||||
14 external file attributes 4 bytes
|
||||
15 relative offset of local header 4 bytes
|
||||
"""
|
||||
def __init__(self, raw_bin):
|
||||
rec = struct.unpack('<6H3I5H2I', raw_bin)
|
||||
self.name_len = rec[9]
|
||||
self.extra_len = rec[10]
|
||||
self.comment_len = rec[11]
|
||||
|
||||
if (rec[0] & 0xff00) == 0x0300: # UNIX
|
||||
self.mode = rec[14] >> 16
|
||||
else:
|
||||
self.mode = stat.S_IFREG | 0644
|
||||
|
||||
|
||||
class _UngetStream(object):
|
||||
"""File like object to read and rewind a stream.
|
||||
"""
|
||||
def __init__(self, fd):
|
||||
self._fd = fd
|
||||
self._buf = None
|
||||
|
||||
def read(self, size = -1):
|
||||
r = []
|
||||
try:
|
||||
if size >= 0:
|
||||
self._ReadChunk(r, size)
|
||||
else:
|
||||
while True:
|
||||
self._ReadChunk(r, 2048)
|
||||
except EOFError:
|
||||
pass
|
||||
|
||||
if len(r) == 1:
|
||||
return r[0]
|
||||
return ''.join(r)
|
||||
|
||||
def unread(self, buf):
|
||||
b = self._buf
|
||||
if b is None or len(b) == 0:
|
||||
self._buf = buf
|
||||
else:
|
||||
self._buf = buf + b
|
||||
|
||||
def _ReadChunk(self, r, size):
|
||||
b = self._buf
|
||||
try:
|
||||
while size > 0:
|
||||
if b is None or len(b) == 0:
|
||||
b = self._Inflate(self._fd.read(2048))
|
||||
if not b:
|
||||
raise EOFError()
|
||||
continue
|
||||
|
||||
use = min(size, len(b))
|
||||
r.append(b[:use])
|
||||
b = b[use:]
|
||||
size -= use
|
||||
finally:
|
||||
self._buf = b
|
||||
|
||||
def _Inflate(self, b):
|
||||
return b
|
||||
|
||||
|
||||
class _FixedLengthStream(_UngetStream):
|
||||
"""File like object to read a fixed length stream.
|
||||
"""
|
||||
def __init__(self, fd, have):
|
||||
_UngetStream.__init__(self, fd)
|
||||
self._have = have
|
||||
|
||||
def _Inflate(self, b):
|
||||
n = self._have
|
||||
if n == 0:
|
||||
self._fd.unread(b)
|
||||
return None
|
||||
|
||||
if len(b) > n:
|
||||
self._fd.unread(b[n:])
|
||||
b = b[:n]
|
||||
self._have -= len(b)
|
||||
return b
|
||||
|
||||
|
||||
class _InflateStream(_UngetStream):
|
||||
"""Inflates the stream as it reads input.
|
||||
"""
|
||||
def __init__(self, fd):
|
||||
_UngetStream.__init__(self, fd)
|
||||
self._z = zlib.decompressobj(-zlib.MAX_WBITS)
|
||||
|
||||
def _Inflate(self, b):
|
||||
z = self._z
|
||||
if not z:
|
||||
self._fd.unread(b)
|
||||
return None
|
||||
|
||||
b = z.decompress(b)
|
||||
if z.unconsumed_tail != '':
|
||||
self._fd.unread(z.unconsumed_tail)
|
||||
elif z.unused_data != '':
|
||||
self._fd.unread(z.unused_data)
|
||||
self._z = None
|
||||
return b
|
||||
72
manifest.py
72
manifest.py
@ -18,8 +18,6 @@ import sys
|
||||
import xml.dom.minidom
|
||||
|
||||
from git_config import GitConfig, IsId
|
||||
from import_tar import ImportTar
|
||||
from import_zip import ImportZip
|
||||
from project import Project, MetaProject, R_TAGS
|
||||
from remote import Remote
|
||||
from error import ManifestParseError
|
||||
@ -245,78 +243,8 @@ class Manifest(object):
|
||||
elif n.nodeName == 'copyfile':
|
||||
self._ParseCopyFile(project, n)
|
||||
|
||||
to_resolve = []
|
||||
by_version = {}
|
||||
|
||||
for n in node.childNodes:
|
||||
if n.nodeName == 'import':
|
||||
self._ParseImport(project, n, to_resolve, by_version)
|
||||
|
||||
for pair in to_resolve:
|
||||
sn, pr = pair
|
||||
try:
|
||||
sn.SetParent(by_version[pr].commit)
|
||||
except KeyError:
|
||||
raise ManifestParseError, \
|
||||
'snapshot %s not in project %s in %s' % \
|
||||
(pr, project.name, self.manifestFile)
|
||||
|
||||
return project
|
||||
|
||||
def _ParseImport(self, project, import_node, to_resolve, by_version):
|
||||
first_url = None
|
||||
for node in import_node.childNodes:
|
||||
if node.nodeName == 'mirror':
|
||||
first_url = self._reqatt(node, 'url')
|
||||
break
|
||||
if not first_url:
|
||||
raise ManifestParseError, \
|
||||
'mirror url required for project %s in %s' % \
|
||||
(project.name, self.manifestFile)
|
||||
|
||||
imp = None
|
||||
for cls in [ImportTar, ImportZip]:
|
||||
if cls.CanAccept(first_url):
|
||||
imp = cls()
|
||||
break
|
||||
if not imp:
|
||||
raise ManifestParseError, \
|
||||
'snapshot %s unsupported for project %s in %s' % \
|
||||
(first_url, project.name, self.manifestFile)
|
||||
|
||||
imp.SetProject(project)
|
||||
|
||||
for node in import_node.childNodes:
|
||||
if node.nodeName == 'remap':
|
||||
old = node.getAttribute('strip')
|
||||
new = node.getAttribute('insert')
|
||||
imp.RemapPath(old, new)
|
||||
|
||||
elif node.nodeName == 'mirror':
|
||||
imp.AddUrl(self._reqatt(node, 'url'))
|
||||
|
||||
for node in import_node.childNodes:
|
||||
if node.nodeName == 'snapshot':
|
||||
sn = imp.Clone()
|
||||
sn.SetVersion(self._reqatt(node, 'version'))
|
||||
sn.SetCommit(node.getAttribute('check'))
|
||||
|
||||
pr = node.getAttribute('prior')
|
||||
if pr:
|
||||
if IsId(pr):
|
||||
sn.SetParent(pr)
|
||||
else:
|
||||
to_resolve.append((sn, pr))
|
||||
|
||||
rev = R_TAGS + sn.TagName
|
||||
|
||||
if rev in project.snapshots:
|
||||
raise ManifestParseError, \
|
||||
'duplicate snapshot %s for project %s in %s' % \
|
||||
(sn.version, project.name, self.manifestFile)
|
||||
project.snapshots[rev] = sn
|
||||
by_version[sn.version] = sn
|
||||
|
||||
def _ParseCopyFile(self, project, node):
|
||||
src = self._reqatt(node, 'src')
|
||||
dest = self._reqatt(node, 'dest')
|
||||
|
||||
@ -1,169 +0,0 @@
|
||||
#
|
||||
# Copyright (C) 2008 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from command import Command
|
||||
from error import GitError, NoSuchProjectError
|
||||
from git_config import IsId
|
||||
from import_tar import ImportTar
|
||||
from import_zip import ImportZip
|
||||
from project import Project
|
||||
from remote import Remote
|
||||
|
||||
def _ToCommit(project, rev):
|
||||
return project.bare_git.rev_parse('--verify', '%s^0' % rev)
|
||||
|
||||
def _Missing(project, rev):
|
||||
return project._revlist('--objects', rev, '--not', '--all')
|
||||
|
||||
|
||||
class ComputeSnapshotCheck(Command):
|
||||
common = False
|
||||
helpSummary = "Compute the check value for a new snapshot"
|
||||
helpUsage = """
|
||||
%prog -p NAME -v VERSION -s FILE [options]
|
||||
"""
|
||||
helpDescription = """
|
||||
%prog computes and then displays the proper check value for a
|
||||
snapshot, so it can be pasted into the manifest file for a project.
|
||||
"""
|
||||
|
||||
def _Options(self, p):
|
||||
g = p.add_option_group('Snapshot description options')
|
||||
g.add_option('-p', '--project',
|
||||
dest='project', metavar='NAME',
|
||||
help='destination project name')
|
||||
g.add_option('-v', '--version',
|
||||
dest='version', metavar='VERSION',
|
||||
help='upstream version/revision identifier')
|
||||
g.add_option('-s', '--snapshot',
|
||||
dest='snapshot', metavar='PATH',
|
||||
help='local tarball path')
|
||||
g.add_option('--new-project',
|
||||
dest='new_project', action='store_true',
|
||||
help='destinition is a new project')
|
||||
g.add_option('--keep',
|
||||
dest='keep_git', action='store_true',
|
||||
help='keep the temporary git repository')
|
||||
|
||||
g = p.add_option_group('Base revision grafting options')
|
||||
g.add_option('--prior',
|
||||
dest='prior', metavar='COMMIT',
|
||||
help='prior revision checksum')
|
||||
|
||||
g = p.add_option_group('Path mangling options')
|
||||
g.add_option('--strip-prefix',
|
||||
dest='strip_prefix', metavar='PREFIX',
|
||||
help='remove prefix from all paths on import')
|
||||
g.add_option('--insert-prefix',
|
||||
dest='insert_prefix', metavar='PREFIX',
|
||||
help='insert prefix before all paths on import')
|
||||
|
||||
|
||||
def _Compute(self, opt):
|
||||
try:
|
||||
real_project = self.GetProjects([opt.project])[0]
|
||||
except NoSuchProjectError:
|
||||
if opt.new_project:
|
||||
print >>sys.stderr, \
|
||||
"warning: project '%s' does not exist" % opt.project
|
||||
else:
|
||||
raise NoSuchProjectError(opt.project)
|
||||
|
||||
self._tmpdir = tempfile.mkdtemp()
|
||||
project = Project(manifest = self.manifest,
|
||||
name = opt.project,
|
||||
remote = Remote('origin'),
|
||||
gitdir = os.path.join(self._tmpdir, '.git'),
|
||||
worktree = self._tmpdir,
|
||||
relpath = opt.project,
|
||||
revision = 'refs/heads/master')
|
||||
project._InitGitDir()
|
||||
|
||||
url = 'file://%s' % os.path.abspath(opt.snapshot)
|
||||
|
||||
imp = None
|
||||
for cls in [ImportTar, ImportZip]:
|
||||
if cls.CanAccept(url):
|
||||
imp = cls()
|
||||
break
|
||||
if not imp:
|
||||
print >>sys.stderr, 'error: %s unsupported' % opt.snapshot
|
||||
sys.exit(1)
|
||||
|
||||
imp.SetProject(project)
|
||||
imp.SetVersion(opt.version)
|
||||
imp.AddUrl(url)
|
||||
|
||||
if opt.prior:
|
||||
if opt.new_project:
|
||||
if not IsId(opt.prior):
|
||||
print >>sys.stderr, 'error: --prior=%s not valid' % opt.prior
|
||||
sys.exit(1)
|
||||
else:
|
||||
try:
|
||||
opt.prior = _ToCommit(real_project, opt.prior)
|
||||
missing = _Missing(real_project, opt.prior)
|
||||
except GitError, e:
|
||||
print >>sys.stderr,\
|
||||
'error: --prior=%s not valid\n%s' \
|
||||
% (opt.prior, e)
|
||||
sys.exit(1)
|
||||
if missing:
|
||||
print >>sys.stderr,\
|
||||
'error: --prior=%s is valid, but is not reachable' \
|
||||
% opt.prior
|
||||
sys.exit(1)
|
||||
imp.SetParent(opt.prior)
|
||||
|
||||
src = opt.strip_prefix
|
||||
dst = opt.insert_prefix
|
||||
if src or dst:
|
||||
if src is None:
|
||||
src = ''
|
||||
if dst is None:
|
||||
dst = ''
|
||||
imp.RemapPath(src, dst)
|
||||
commitId = imp.Import()
|
||||
|
||||
print >>sys.stderr,"%s\t%s" % (commitId, imp.version)
|
||||
return project
|
||||
|
||||
def Execute(self, opt, args):
|
||||
if args \
|
||||
or not opt.project \
|
||||
or not opt.version \
|
||||
or not opt.snapshot:
|
||||
self.Usage()
|
||||
|
||||
success = False
|
||||
project = None
|
||||
try:
|
||||
self._tmpdir = None
|
||||
project = self._Compute(opt)
|
||||
finally:
|
||||
if project and opt.keep_git:
|
||||
print 'GIT_DIR = %s' % (project.gitdir)
|
||||
elif self._tmpdir:
|
||||
for root, dirs, files in os.walk(self._tmpdir, topdown=False):
|
||||
for name in files:
|
||||
os.remove(os.path.join(root, name))
|
||||
for name in dirs:
|
||||
os.rmdir(os.path.join(root, name))
|
||||
os.rmdir(self._tmpdir)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user