From df830f12389984adeeba04659615b9fc230f2db3 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 30 Oct 2008 09:21:43 -0700 Subject: [PATCH] Remove import_tar, import_zip and the elements Now that repo relies only on the git data stream (as it is much faster to download through) we don't really need to be parsing the elements within manifest. Its a lot of complex code to convert the tar (or zip) through to a fast import stream, and we just aren't calling it anymore. Signed-off-by: Shawn O. Pearce --- import_ext.py | 422 ------------------------------ import_tar.py | 206 --------------- import_zip.py | 345 ------------------------ manifest.py | 72 ----- subcmds/compute_snapshot_check.py | 169 ------------ 5 files changed, 1214 deletions(-) delete mode 100644 import_ext.py delete mode 100644 import_tar.py delete mode 100644 import_zip.py delete mode 100644 subcmds/compute_snapshot_check.py diff --git a/import_ext.py b/import_ext.py deleted file mode 100644 index 2a1ebf8..0000000 --- a/import_ext.py +++ /dev/null @@ -1,422 +0,0 @@ -# -# Copyright (C) 2008 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random -import stat -import sys -import urllib2 -import StringIO - -from error import GitError, ImportError -from git_command import GitCommand - -class ImportExternal(object): - """Imports a single revision from a non-git data source. - Suitable for use to import a tar or zip based snapshot. - """ - def __init__(self): - self._marks = 0 - self._files = {} - self._tempref = 'refs/repo-external/import' - - self._urls = [] - self._remap = [] - self.parent = None - self._user_name = 'Upstream' - self._user_email = 'upstream-import@none' - self._user_when = 1000000 - - self.commit = None - - def Clone(self): - r = self.__class__() - - r.project = self.project - for u in self._urls: - r._urls.append(u) - for p in self._remap: - r._remap.append(_PathMap(r, p._old, p._new)) - - return r - - def SetProject(self, project): - self.project = project - - def SetVersion(self, version): - self.version = version - - def AddUrl(self, url): - self._urls.append(url) - - def SetParent(self, commit_hash): - self.parent = commit_hash - - def SetCommit(self, commit_hash): - self.commit = commit_hash - - def RemapPath(self, old, new, replace_version=True): - self._remap.append(_PathMap(self, old, new)) - - @property - def TagName(self): - v = '' - for c in self.version: - if c >= '0' and c <= '9': - v += c - elif c >= 'A' and c <= 'Z': - v += c - elif c >= 'a' and c <= 'z': - v += c - elif c in ('-', '_', '.', '/', '+', '@'): - v += c - return 'upstream/%s' % v - - @property - def PackageName(self): - n = self.project.name - if n.startswith('platform/'): - # This was not my finest moment... - # - n = n[len('platform/'):] - return n - - def Import(self): - self._need_graft = False - if self.parent: - try: - self.project.bare_git.cat_file('-e', self.parent) - except GitError: - self._need_graft = True - - gfi = GitCommand(self.project, - ['fast-import', '--force', '--quiet'], - bare = True, - provide_stdin = True) - try: - self._out = gfi.stdin - - try: - self._UnpackFiles() - self._MakeCommit() - self._out.flush() - finally: - rc = gfi.Wait() - if rc != 0: - raise ImportError('fast-import failed') - - if self._need_graft: - id = self._GraftCommit() - else: - id = self.project.bare_git.rev_parse('%s^0' % self._tempref) - - if self.commit and self.commit != id: - raise ImportError('checksum mismatch: %s expected,' - ' %s imported' % (self.commit, id)) - - self._MakeTag(id) - return id - finally: - try: - self.project.bare_git.DeleteRef(self._tempref) - except GitError: - pass - - def _PickUrl(self, failed): - u = map(lambda x: x.replace('%version%', self.version), self._urls) - for f in failed: - if f in u: - u.remove(f) - if len(u) == 0: - return None - return random.choice(u) - - def _OpenUrl(self): - failed = {} - while True: - url = self._PickUrl(failed.keys()) - if url is None: - why = 'Cannot download %s' % self.project.name - - if failed: - why += ': one or more mirrors are down\n' - bad_urls = list(failed.keys()) - bad_urls.sort() - for url in bad_urls: - why += ' %s: %s\n' % (url, failed[url]) - else: - why += ': no mirror URLs' - raise ImportError(why) - - print >>sys.stderr, "Getting %s ..." % url - try: - return urllib2.urlopen(url), url - except urllib2.HTTPError, e: - failed[url] = e.code - except urllib2.URLError, e: - failed[url] = e.reason[1] - except OSError, e: - failed[url] = e.strerror - - def _UnpackFiles(self): - raise NotImplementedError - - def _NextMark(self): - self._marks += 1 - return self._marks - - def _UnpackOneFile(self, mode, size, name, fd): - if stat.S_ISDIR(mode): # directory - return - else: - mode = self._CleanMode(mode, name) - - old_name = name - name = self._CleanName(name) - - if stat.S_ISLNK(mode) and self._remap: - # The link is relative to the old_name, and may need to - # be rewritten according to our remap rules if it goes - # up high enough in the tree structure. - # - dest = self._RewriteLink(fd.read(size), old_name, name) - fd = StringIO.StringIO(dest) - size = len(dest) - - fi = _File(mode, name, self._NextMark()) - - self._out.write('blob\n') - self._out.write('mark :%d\n' % fi.mark) - self._out.write('data %d\n' % size) - while size > 0: - n = min(2048, size) - self._out.write(fd.read(n)) - size -= n - self._out.write('\n') - self._files[fi.name] = fi - - def _SetFileMode(self, name, mode): - if not stat.S_ISDIR(mode): - mode = self._CleanMode(mode, name) - name = self._CleanName(name) - try: - fi = self._files[name] - except KeyError: - raise ImportError('file %s was not unpacked' % name) - fi.mode = mode - - def _RewriteLink(self, dest, relto_old, relto_new): - # Drop the last components of the symlink itself - # as the dest is relative to the directory its in. - # - relto_old = _TrimPath(relto_old) - relto_new = _TrimPath(relto_new) - - # Resolve the link to be absolute from the top of - # the archive, so we can remap its destination. - # - while dest.find('/./') >= 0 or dest.find('//') >= 0: - dest = dest.replace('/./', '/') - dest = dest.replace('//', '/') - - if dest.startswith('../') or dest.find('/../') > 0: - dest = _FoldPath('%s/%s' % (relto_old, dest)) - - for pm in self._remap: - if pm.Matches(dest): - dest = pm.Apply(dest) - break - - dest, relto_new = _StripCommonPrefix(dest, relto_new) - while relto_new: - i = relto_new.find('/') - if i > 0: - relto_new = relto_new[i + 1:] - else: - relto_new = '' - dest = '../' + dest - return dest - - def _CleanMode(self, mode, name): - if stat.S_ISREG(mode): # regular file - if (mode & 0111) == 0: - return 0644 - else: - return 0755 - elif stat.S_ISLNK(mode): # symlink - return stat.S_IFLNK - else: - raise ImportError('invalid mode %o in %s' % (mode, name)) - - def _CleanName(self, name): - old_name = name - for pm in self._remap: - if pm.Matches(name): - name = pm.Apply(name) - break - while name.startswith('/'): - name = name[1:] - if not name: - raise ImportError('path %s is empty after remap' % old_name) - if name.find('/./') >= 0 or name.find('/../') >= 0: - raise ImportError('path %s contains relative parts' % name) - return name - - def _MakeCommit(self): - msg = '%s %s\n' % (self.PackageName, self.version) - - self._out.write('commit %s\n' % self._tempref) - self._out.write('committer %s <%s> %d +0000\n' % ( - self._user_name, - self._user_email, - self._user_when)) - self._out.write('data %d\n' % len(msg)) - self._out.write(msg) - self._out.write('\n') - if self.parent and not self._need_graft: - self._out.write('from %s^0\n' % self.parent) - self._out.write('deleteall\n') - - for f in self._files.values(): - self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name)) - self._out.write('\n') - - def _GraftCommit(self): - raw = self.project.bare_git.cat_file('commit', self._tempref) - raw = raw.split("\n") - while raw[1].startswith('parent '): - del raw[1] - raw.insert(1, 'parent %s' % self.parent) - id = self._WriteObject('commit', "\n".join(raw)) - - graft_file = os.path.join(self.project.gitdir, 'info/grafts') - if os.path.exists(graft_file): - graft_list = open(graft_file, 'rb').read().split("\n") - if graft_list and graft_list[-1] == '': - del graft_list[-1] - else: - graft_list = [] - - exists = False - for line in graft_list: - if line == id: - exists = True - break - - if not exists: - graft_list.append(id) - graft_list.append('') - fd = open(graft_file, 'wb') - fd.write("\n".join(graft_list)) - fd.close() - - return id - - def _MakeTag(self, id): - name = self.TagName - - raw = [] - raw.append('object %s' % id) - raw.append('type commit') - raw.append('tag %s' % name) - raw.append('tagger %s <%s> %d +0000' % ( - self._user_name, - self._user_email, - self._user_when)) - raw.append('') - raw.append('%s %s\n' % (self.PackageName, self.version)) - - tagid = self._WriteObject('tag', "\n".join(raw)) - self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid) - - def _WriteObject(self, type, data): - wo = GitCommand(self.project, - ['hash-object', '-t', type, '-w', '--stdin'], - bare = True, - provide_stdin = True, - capture_stdout = True, - capture_stderr = True) - wo.stdin.write(data) - if wo.Wait() != 0: - raise GitError('cannot create %s from (%s)' % (type, data)) - return wo.stdout[:-1] - - -def _TrimPath(path): - i = path.rfind('/') - if i > 0: - path = path[0:i] - return '' - -def _StripCommonPrefix(a, b): - while True: - ai = a.find('/') - bi = b.find('/') - if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]: - a = a[ai + 1:] - b = b[bi + 1:] - else: - break - return a, b - -def _FoldPath(path): - while True: - if path.startswith('../'): - return path - - i = path.find('/../') - if i <= 0: - if path.startswith('/'): - return path[1:] - return path - - lhs = path[0:i] - rhs = path[i + 4:] - - i = lhs.rfind('/') - if i > 0: - path = lhs[0:i + 1] + rhs - else: - path = rhs - -class _File(object): - def __init__(self, mode, name, mark): - self.mode = mode - self.name = name - self.mark = mark - - -class _PathMap(object): - def __init__(self, imp, old, new): - self._imp = imp - self._old = old - self._new = new - - def _r(self, p): - return p.replace('%version%', self._imp.version) - - @property - def old(self): - return self._r(self._old) - - @property - def new(self): - return self._r(self._new) - - def Matches(self, name): - return name.startswith(self.old) - - def Apply(self, name): - return self.new + name[len(self.old):] diff --git a/import_tar.py b/import_tar.py deleted file mode 100644 index d7ce14d..0000000 --- a/import_tar.py +++ /dev/null @@ -1,206 +0,0 @@ -# -# Copyright (C) 2008 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import bz2 -import stat -import tarfile -import zlib -import StringIO - -from import_ext import ImportExternal -from error import ImportError - -class ImportTar(ImportExternal): - """Streams a (optionally compressed) tar file from the network - directly into a Project's Git repository. - """ - @classmethod - def CanAccept(cls, url): - """Can this importer read and unpack the data stored at url? - """ - if url.endswith('.tar.gz') or url.endswith('.tgz'): - return True - if url.endswith('.tar.bz2'): - return True - if url.endswith('.tar'): - return True - return False - - def _UnpackFiles(self): - url_fd, url = self._OpenUrl() - try: - if url.endswith('.tar.gz') or url.endswith('.tgz'): - tar_fd = _Gzip(url_fd) - elif url.endswith('.tar.bz2'): - tar_fd = _Bzip2(url_fd) - elif url.endswith('.tar'): - tar_fd = _Raw(url_fd) - else: - raise ImportError('non-tar file extension: %s' % url) - - try: - tar = tarfile.TarFile(name = url, - mode = 'r', - fileobj = tar_fd) - try: - for entry in tar: - mode = entry.mode - - if (mode & 0170000) == 0: - if entry.isdir(): - mode |= stat.S_IFDIR - elif entry.isfile() or entry.islnk(): # hard links as files - mode |= stat.S_IFREG - elif entry.issym(): - mode |= stat.S_IFLNK - - if stat.S_ISLNK(mode): # symlink - data_fd = StringIO.StringIO(entry.linkname) - data_sz = len(entry.linkname) - elif stat.S_ISDIR(mode): # directory - data_fd = StringIO.StringIO('') - data_sz = 0 - else: - data_fd = tar.extractfile(entry) - data_sz = entry.size - - self._UnpackOneFile(mode, data_sz, entry.name, data_fd) - finally: - tar.close() - finally: - tar_fd.close() - finally: - url_fd.close() - - - -class _DecompressStream(object): - """file like object to decompress a tar stream - """ - def __init__(self, fd): - self._fd = fd - self._pos = 0 - self._buf = None - - def tell(self): - return self._pos - - def seek(self, offset): - d = offset - self._pos - if d > 0: - self.read(d) - elif d == 0: - pass - else: - raise NotImplementedError, 'seek backwards' - - def close(self): - self._fd = None - - def read(self, size = -1): - if not self._fd: - raise EOFError, 'Reached EOF' - - r = [] - try: - if size >= 0: - self._ReadChunk(r, size) - else: - while True: - self._ReadChunk(r, 2048) - except EOFError: - pass - - if len(r) == 1: - r = r[0] - else: - r = ''.join(r) - self._pos += len(r) - return r - - def _ReadChunk(self, r, size): - b = self._buf - try: - while size > 0: - if b is None or len(b) == 0: - b = self._Decompress(self._fd.read(2048)) - continue - - use = min(size, len(b)) - r.append(b[:use]) - b = b[use:] - size -= use - finally: - self._buf = b - - def _Decompress(self, b): - raise NotImplementedError, '_Decompress' - - -class _Raw(_DecompressStream): - """file like object for an uncompressed stream - """ - def __init__(self, fd): - _DecompressStream.__init__(self, fd) - - def _Decompress(self, b): - return b - - -class _Bzip2(_DecompressStream): - """file like object to decompress a .bz2 stream - """ - def __init__(self, fd): - _DecompressStream.__init__(self, fd) - self._bz = bz2.BZ2Decompressor() - - def _Decompress(self, b): - return self._bz.decompress(b) - - -_FHCRC, _FEXTRA, _FNAME, _FCOMMENT = 2, 4, 8, 16 -class _Gzip(_DecompressStream): - """file like object to decompress a .gz stream - """ - def __init__(self, fd): - _DecompressStream.__init__(self, fd) - self._z = zlib.decompressobj(-zlib.MAX_WBITS) - - magic = fd.read(2) - if magic != '\037\213': - raise IOError, 'Not a gzipped file' - - method = ord(fd.read(1)) - if method != 8: - raise IOError, 'Unknown compression method' - - flag = ord(fd.read(1)) - fd.read(6) - - if flag & _FEXTRA: - xlen = ord(fd.read(1)) - xlen += 256 * ord(fd.read(1)) - fd.read(xlen) - if flag & _FNAME: - while fd.read(1) != '\0': - pass - if flag & _FCOMMENT: - while fd.read(1) != '\0': - pass - if flag & _FHCRC: - fd.read(2) - - def _Decompress(self, b): - return self._z.decompress(b) diff --git a/import_zip.py b/import_zip.py deleted file mode 100644 index 08aff32..0000000 --- a/import_zip.py +++ /dev/null @@ -1,345 +0,0 @@ -# -# Copyright (C) 2008 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import stat -import struct -import zlib -import cStringIO - -from import_ext import ImportExternal -from error import ImportError - -class ImportZip(ImportExternal): - """Streams a zip file from the network directly into a Project's - Git repository. - """ - @classmethod - def CanAccept(cls, url): - """Can this importer read and unpack the data stored at url? - """ - if url.endswith('.zip') or url.endswith('.jar'): - return True - return False - - def _UnpackFiles(self): - url_fd, url = self._OpenUrl() - try: - if not self.__class__.CanAccept(url): - raise ImportError('non-zip file extension: %s' % url) - - zip = _ZipFile(url_fd) - for entry in zip.FileRecords(): - data = zip.Open(entry).read() - sz = len(data) - - if data and _SafeCRLF(data): - data = data.replace('\r\n', '\n') - sz = len(data) - - fd = cStringIO.StringIO(data) - self._UnpackOneFile(entry.mode, sz, entry.name, fd) - zip.Close(entry) - - for entry in zip.CentralDirectory(): - self._SetFileMode(entry.name, entry.mode) - - zip.CheckTail() - finally: - url_fd.close() - - -def _SafeCRLF(data): - """Is it reasonably safe to perform a CRLF->LF conversion? - - If the stream contains a NUL byte it is likely binary, - and thus a CRLF->LF conversion may damage the stream. - - If the only NUL is in the last position of the stream, - but it otherwise can do a CRLF<->LF conversion we do - the CRLF conversion anyway. At least one source ZIP - file has this structure in its source code. - - If every occurrance of a CR and LF is paired up as a - CRLF pair then the conversion is safely bi-directional. - s/\r\n/\n/g == s/\n/\r\\n/g can convert between them. - """ - nul = data.find('\0') - if 0 <= nul and nul < (len(data) - 1): - return False - - n_lf = 0 - last = 0 - while True: - lf = data.find('\n', last) - if lf < 0: - break - if lf == 0 or data[lf - 1] != '\r': - return False - last = lf + 1 - n_lf += 1 - return n_lf > 0 - -class _ZipFile(object): - """Streaming iterator to parse a zip file on the fly. - """ - def __init__(self, fd): - self._fd = _UngetStream(fd) - - def FileRecords(self): - return _FileIter(self._fd) - - def CentralDirectory(self): - return _CentIter(self._fd) - - def CheckTail(self): - type_buf = self._fd.read(4) - type = struct.unpack('> 16 - else: - self.mode = stat.S_IFREG | 0644 - - -class _UngetStream(object): - """File like object to read and rewind a stream. - """ - def __init__(self, fd): - self._fd = fd - self._buf = None - - def read(self, size = -1): - r = [] - try: - if size >= 0: - self._ReadChunk(r, size) - else: - while True: - self._ReadChunk(r, 2048) - except EOFError: - pass - - if len(r) == 1: - return r[0] - return ''.join(r) - - def unread(self, buf): - b = self._buf - if b is None or len(b) == 0: - self._buf = buf - else: - self._buf = buf + b - - def _ReadChunk(self, r, size): - b = self._buf - try: - while size > 0: - if b is None or len(b) == 0: - b = self._Inflate(self._fd.read(2048)) - if not b: - raise EOFError() - continue - - use = min(size, len(b)) - r.append(b[:use]) - b = b[use:] - size -= use - finally: - self._buf = b - - def _Inflate(self, b): - return b - - -class _FixedLengthStream(_UngetStream): - """File like object to read a fixed length stream. - """ - def __init__(self, fd, have): - _UngetStream.__init__(self, fd) - self._have = have - - def _Inflate(self, b): - n = self._have - if n == 0: - self._fd.unread(b) - return None - - if len(b) > n: - self._fd.unread(b[n:]) - b = b[:n] - self._have -= len(b) - return b - - -class _InflateStream(_UngetStream): - """Inflates the stream as it reads input. - """ - def __init__(self, fd): - _UngetStream.__init__(self, fd) - self._z = zlib.decompressobj(-zlib.MAX_WBITS) - - def _Inflate(self, b): - z = self._z - if not z: - self._fd.unread(b) - return None - - b = z.decompress(b) - if z.unconsumed_tail != '': - self._fd.unread(z.unconsumed_tail) - elif z.unused_data != '': - self._fd.unread(z.unused_data) - self._z = None - return b diff --git a/manifest.py b/manifest.py index 8c5a8d6..ffff14a 100644 --- a/manifest.py +++ b/manifest.py @@ -18,8 +18,6 @@ import sys import xml.dom.minidom from git_config import GitConfig, IsId -from import_tar import ImportTar -from import_zip import ImportZip from project import Project, MetaProject, R_TAGS from remote import Remote from error import ManifestParseError @@ -245,78 +243,8 @@ class Manifest(object): elif n.nodeName == 'copyfile': self._ParseCopyFile(project, n) - to_resolve = [] - by_version = {} - - for n in node.childNodes: - if n.nodeName == 'import': - self._ParseImport(project, n, to_resolve, by_version) - - for pair in to_resolve: - sn, pr = pair - try: - sn.SetParent(by_version[pr].commit) - except KeyError: - raise ManifestParseError, \ - 'snapshot %s not in project %s in %s' % \ - (pr, project.name, self.manifestFile) - return project - def _ParseImport(self, project, import_node, to_resolve, by_version): - first_url = None - for node in import_node.childNodes: - if node.nodeName == 'mirror': - first_url = self._reqatt(node, 'url') - break - if not first_url: - raise ManifestParseError, \ - 'mirror url required for project %s in %s' % \ - (project.name, self.manifestFile) - - imp = None - for cls in [ImportTar, ImportZip]: - if cls.CanAccept(first_url): - imp = cls() - break - if not imp: - raise ManifestParseError, \ - 'snapshot %s unsupported for project %s in %s' % \ - (first_url, project.name, self.manifestFile) - - imp.SetProject(project) - - for node in import_node.childNodes: - if node.nodeName == 'remap': - old = node.getAttribute('strip') - new = node.getAttribute('insert') - imp.RemapPath(old, new) - - elif node.nodeName == 'mirror': - imp.AddUrl(self._reqatt(node, 'url')) - - for node in import_node.childNodes: - if node.nodeName == 'snapshot': - sn = imp.Clone() - sn.SetVersion(self._reqatt(node, 'version')) - sn.SetCommit(node.getAttribute('check')) - - pr = node.getAttribute('prior') - if pr: - if IsId(pr): - sn.SetParent(pr) - else: - to_resolve.append((sn, pr)) - - rev = R_TAGS + sn.TagName - - if rev in project.snapshots: - raise ManifestParseError, \ - 'duplicate snapshot %s for project %s in %s' % \ - (sn.version, project.name, self.manifestFile) - project.snapshots[rev] = sn - by_version[sn.version] = sn - def _ParseCopyFile(self, project, node): src = self._reqatt(node, 'src') dest = self._reqatt(node, 'dest') diff --git a/subcmds/compute_snapshot_check.py b/subcmds/compute_snapshot_check.py deleted file mode 100644 index 82db359..0000000 --- a/subcmds/compute_snapshot_check.py +++ /dev/null @@ -1,169 +0,0 @@ -# -# Copyright (C) 2008 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import tempfile - -from command import Command -from error import GitError, NoSuchProjectError -from git_config import IsId -from import_tar import ImportTar -from import_zip import ImportZip -from project import Project -from remote import Remote - -def _ToCommit(project, rev): - return project.bare_git.rev_parse('--verify', '%s^0' % rev) - -def _Missing(project, rev): - return project._revlist('--objects', rev, '--not', '--all') - - -class ComputeSnapshotCheck(Command): - common = False - helpSummary = "Compute the check value for a new snapshot" - helpUsage = """ -%prog -p NAME -v VERSION -s FILE [options] -""" - helpDescription = """ -%prog computes and then displays the proper check value for a -snapshot, so it can be pasted into the manifest file for a project. -""" - - def _Options(self, p): - g = p.add_option_group('Snapshot description options') - g.add_option('-p', '--project', - dest='project', metavar='NAME', - help='destination project name') - g.add_option('-v', '--version', - dest='version', metavar='VERSION', - help='upstream version/revision identifier') - g.add_option('-s', '--snapshot', - dest='snapshot', metavar='PATH', - help='local tarball path') - g.add_option('--new-project', - dest='new_project', action='store_true', - help='destinition is a new project') - g.add_option('--keep', - dest='keep_git', action='store_true', - help='keep the temporary git repository') - - g = p.add_option_group('Base revision grafting options') - g.add_option('--prior', - dest='prior', metavar='COMMIT', - help='prior revision checksum') - - g = p.add_option_group('Path mangling options') - g.add_option('--strip-prefix', - dest='strip_prefix', metavar='PREFIX', - help='remove prefix from all paths on import') - g.add_option('--insert-prefix', - dest='insert_prefix', metavar='PREFIX', - help='insert prefix before all paths on import') - - - def _Compute(self, opt): - try: - real_project = self.GetProjects([opt.project])[0] - except NoSuchProjectError: - if opt.new_project: - print >>sys.stderr, \ - "warning: project '%s' does not exist" % opt.project - else: - raise NoSuchProjectError(opt.project) - - self._tmpdir = tempfile.mkdtemp() - project = Project(manifest = self.manifest, - name = opt.project, - remote = Remote('origin'), - gitdir = os.path.join(self._tmpdir, '.git'), - worktree = self._tmpdir, - relpath = opt.project, - revision = 'refs/heads/master') - project._InitGitDir() - - url = 'file://%s' % os.path.abspath(opt.snapshot) - - imp = None - for cls in [ImportTar, ImportZip]: - if cls.CanAccept(url): - imp = cls() - break - if not imp: - print >>sys.stderr, 'error: %s unsupported' % opt.snapshot - sys.exit(1) - - imp.SetProject(project) - imp.SetVersion(opt.version) - imp.AddUrl(url) - - if opt.prior: - if opt.new_project: - if not IsId(opt.prior): - print >>sys.stderr, 'error: --prior=%s not valid' % opt.prior - sys.exit(1) - else: - try: - opt.prior = _ToCommit(real_project, opt.prior) - missing = _Missing(real_project, opt.prior) - except GitError, e: - print >>sys.stderr,\ - 'error: --prior=%s not valid\n%s' \ - % (opt.prior, e) - sys.exit(1) - if missing: - print >>sys.stderr,\ - 'error: --prior=%s is valid, but is not reachable' \ - % opt.prior - sys.exit(1) - imp.SetParent(opt.prior) - - src = opt.strip_prefix - dst = opt.insert_prefix - if src or dst: - if src is None: - src = '' - if dst is None: - dst = '' - imp.RemapPath(src, dst) - commitId = imp.Import() - - print >>sys.stderr,"%s\t%s" % (commitId, imp.version) - return project - - def Execute(self, opt, args): - if args \ - or not opt.project \ - or not opt.version \ - or not opt.snapshot: - self.Usage() - - success = False - project = None - try: - self._tmpdir = None - project = self._Compute(opt) - finally: - if project and opt.keep_git: - print 'GIT_DIR = %s' % (project.gitdir) - elif self._tmpdir: - for root, dirs, files in os.walk(self._tmpdir, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - os.rmdir(self._tmpdir) -