Initial Contribution

import_ext.py

@@ -0,0 +1,422 @@
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import random
+import stat
+import sys
+import urllib2
+import StringIO
+
+from error import GitError, ImportError
+from git_command import GitCommand
+
+class ImportExternal(object):
+  """Imports a single revision from a non-git data source.
+     Suitable for use to import a tar or zip based snapshot.
+  """
+  def __init__(self):
+    self._marks = 0
+    self._files = {}
+    self._tempref = 'refs/repo-external/import'
+
+    self._urls = []
+    self._remap = []
+    self.parent = None
+    self._user_name = 'Upstream'
+    self._user_email = 'upstream-import@none'
+    self._user_when = 1000000
+
+    self.commit = None
+
+  def Clone(self):
+    r = self.__class__()
+
+    r.project = self.project
+    for u in self._urls:
+      r._urls.append(u)
+    for p in self._remap:
+      r._remap.append(_PathMap(r, p._old, p._new))
+
+    return r
+
+  def SetProject(self, project):
+    self.project = project
+
+  def SetVersion(self, version):
+    self.version = version
+
+  def AddUrl(self, url):
+    self._urls.append(url)
+
+  def SetParent(self, commit_hash):
+    self.parent = commit_hash
+
+  def SetCommit(self, commit_hash):
+    self.commit = commit_hash
+
+  def RemapPath(self, old, new, replace_version=True):
+    self._remap.append(_PathMap(self, old, new))
+
+  @property
+  def TagName(self):
+    v = ''
+    for c in self.version:
+      if c >= '0' and c <= '9':
+        v += c
+      elif c >= 'A' and c <= 'Z':
+        v += c
+      elif c >= 'a' and c <= 'z':
+        v += c
+      elif c in ('-', '_', '.', '/', '+', '@'):
+        v += c
+    return 'upstream/%s' % v
+
+  @property
+  def PackageName(self):
+    n = self.project.name
+    if n.startswith('platform/'):
+      # This was not my finest moment...
+      #
+      n = n[len('platform/'):]
+    return n
+
+  def Import(self):
+    self._need_graft = False
+    if self.parent:
+      try:
+        self.project.bare_git.cat_file('-e', self.parent)
+      except GitError:
+        self._need_graft = True
+
+    gfi = GitCommand(self.project,
+                     ['fast-import', '--force', '--quiet'],
+                     bare = True,
+                     provide_stdin = True)
+    try:
+      self._out = gfi.stdin
+
+      try:
+        self._UnpackFiles()
+        self._MakeCommit()
+        self._out.flush()
+      finally:
+        rc = gfi.Wait()
+      if rc != 0:
+        raise ImportError('fast-import failed')
+
+      if self._need_graft:
+        id = self._GraftCommit()
+      else:
+        id = self.project.bare_git.rev_parse('%s^0' % self._tempref)
+
+      if self.commit and self.commit != id:
+        raise ImportError('checksum mismatch: %s expected,'
+                          ' %s imported' % (self.commit, id))
+
+      self._MakeTag(id)
+      return id
+    finally:
+      try:
+        self.project.bare_git.DeleteRef(self._tempref)
+      except GitError:
+        pass
+
+  def _PickUrl(self, failed):
+    u = map(lambda x: x.replace('%version%', self.version), self._urls)
+    for f in failed:
+      if f in u:
+        u.remove(f)
+    if len(u) == 0:
+      return None
+    return random.choice(u)
+
+  def _OpenUrl(self):
+    failed = {}
+    while True:
+      url = self._PickUrl(failed.keys())
+      if url is None:
+        why = 'Cannot download %s' % self.project.name
+
+        if failed:
+          why += ': one or more mirrors are down\n'
+          bad_urls = list(failed.keys())
+          bad_urls.sort()
+          for url in bad_urls:
+            why += '  %s: %s\n' % (url, failed[url])
+        else:
+          why += ': no mirror URLs'
+        raise ImportError(why)
+
+      print >>sys.stderr, "Getting %s ..." % url
+      try:
+        return urllib2.urlopen(url), url
+      except urllib2.HTTPError, e:
+        failed[url] = e.code
+      except urllib2.URLError, e:
+        failed[url] = e.reason[1]
+      except OSError, e:
+        failed[url] = e.strerror
+
+  def _UnpackFiles(self):
+    raise NotImplementedError
+
+  def _NextMark(self):
+    self._marks += 1
+    return self._marks
+
+  def _UnpackOneFile(self, mode, size, name, fd):
+    if stat.S_ISDIR(mode):    # directory
+      return
+    else:
+      mode = self._CleanMode(mode, name)
+
+    old_name = name
+    name = self._CleanName(name)
+
+    if stat.S_ISLNK(mode) and self._remap:
+      # The link is relative to the old_name, and may need to
+      # be rewritten according to our remap rules if it goes
+      # up high enough in the tree structure.
+      #
+      dest = self._RewriteLink(fd.read(size), old_name, name)
+      fd = StringIO.StringIO(dest)
+      size = len(dest)
+
+    fi = _File(mode, name, self._NextMark())
+
+    self._out.write('blob\n')
+    self._out.write('mark :%d\n' % fi.mark)
+    self._out.write('data %d\n' % size)
+    while size > 0:
+      n = min(2048, size)
+      self._out.write(fd.read(n))
+      size -= n
+    self._out.write('\n')
+    self._files[fi.name] = fi
+
+  def _SetFileMode(self, name, mode):
+    if not stat.S_ISDIR(mode):
+      mode = self._CleanMode(mode, name)
+      name = self._CleanName(name)
+      try:
+        fi = self._files[name]
+      except KeyError:
+        raise ImportError('file %s was not unpacked' % name)
+      fi.mode = mode
+
+  def _RewriteLink(self, dest, relto_old, relto_new):
+    # Drop the last components of the symlink itself
+    # as the dest is relative to the directory its in.
+    #
+    relto_old = _TrimPath(relto_old)
+    relto_new = _TrimPath(relto_new)
+
+    # Resolve the link to be absolute from the top of
+    # the archive, so we can remap its destination.
+    #
+    while dest.find('/./') >= 0 or dest.find('//') >= 0:
+      dest = dest.replace('/./', '/')
+      dest = dest.replace('//', '/')
+
+    if dest.startswith('../') or dest.find('/../') > 0:
+      dest = _FoldPath('%s/%s' % (relto_old, dest))
+
+    for pm in self._remap:
+      if pm.Matches(dest):
+        dest = pm.Apply(dest)
+        break
+
+    dest, relto_new = _StripCommonPrefix(dest, relto_new)
+    while relto_new:
+      i = relto_new.find('/')
+      if i > 0:
+        relto_new = relto_new[i + 1:]
+      else:
+        relto_new = ''
+      dest = '../' + dest
+    return dest
+
+  def _CleanMode(self, mode, name):
+    if stat.S_ISREG(mode):  # regular file
+      if (mode & 0111) == 0:
+        return 0644
+      else:
+        return 0755
+    elif stat.S_ISLNK(mode):  # symlink
+      return stat.S_IFLNK
+    else:
+      raise ImportError('invalid mode %o in %s' % (mode, name))
+
+  def _CleanName(self, name):
+    old_name = name
+    for pm in self._remap:
+      if pm.Matches(name):
+        name = pm.Apply(name)
+        break
+    while name.startswith('/'):
+      name = name[1:]
+    if not name:
+      raise ImportError('path %s is empty after remap' % old_name)
+    if name.find('/./') >= 0 or name.find('/../') >= 0:
+      raise ImportError('path %s contains relative parts' % name)
+    return name
+
+  def _MakeCommit(self):
+    msg = '%s %s\n' % (self.PackageName, self.version)
+
+    self._out.write('commit %s\n' % self._tempref)
+    self._out.write('committer %s <%s> %d +0000\n' % (
+                    self._user_name,
+                    self._user_email,
+                    self._user_when))
+    self._out.write('data %d\n' % len(msg))
+    self._out.write(msg)
+    self._out.write('\n')
+    if self.parent and not self._need_graft:
+      self._out.write('from %s^0\n' % self.parent)
+      self._out.write('deleteall\n')
+
+    for f in self._files.values():
+      self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name))
+    self._out.write('\n')
+
+  def _GraftCommit(self):
+    raw = self.project.bare_git.cat_file('commit', self._tempref)
+    raw = raw.split("\n")
+    while raw[1].startswith('parent '):
+      del raw[1]
+    raw.insert(1, 'parent %s' % self.parent)
+    id = self._WriteObject('commit', "\n".join(raw))
+
+    graft_file = os.path.join(self.project.gitdir, 'info/grafts')
+    if os.path.exists(graft_file):
+      graft_list = open(graft_file, 'rb').read().split("\n")
+      if graft_list and graft_list[-1] == '':
+        del graft_list[-1]
+    else:
+      graft_list = []
+
+    exists = False
+    for line in graft_list:
+      if line == id:
+        exists = True
+        break
+
+    if not exists:
+      graft_list.append(id)
+      graft_list.append('')
+      fd = open(graft_file, 'wb')
+      fd.write("\n".join(graft_list))
+      fd.close()
+
+    return id
+
+  def _MakeTag(self, id):
+    name = self.TagName
+
+    raw = []
+    raw.append('object %s' % id)
+    raw.append('type commit')
+    raw.append('tag %s' % name)
+    raw.append('tagger %s <%s> %d +0000' % (
+      self._user_name,
+      self._user_email,
+      self._user_when))
+    raw.append('')
+    raw.append('%s %s\n' % (self.PackageName, self.version))
+
+    tagid = self._WriteObject('tag', "\n".join(raw))
+    self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid)
+
+  def _WriteObject(self, type, data):
+    wo = GitCommand(self.project,
+                    ['hash-object', '-t', type, '-w', '--stdin'],
+                    bare = True,
+                    provide_stdin = True,
+                    capture_stdout = True,
+                    capture_stderr = True)
+    wo.stdin.write(data)
+    if wo.Wait() != 0:
+      raise GitError('cannot create %s from (%s)' % (type, data))
+    return wo.stdout[:-1]
+
+
+def _TrimPath(path):
+  i = path.rfind('/')
+  if i > 0:
+    path = path[0:i]
+  return ''
+
+def _StripCommonPrefix(a, b):
+  while True:
+    ai = a.find('/')
+    bi = b.find('/')
+    if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]:
+      a = a[ai + 1:]
+      b = b[bi + 1:]
+    else:
+      break
+  return a, b
+
+def _FoldPath(path):
+  while True:
+    if path.startswith('../'):
+      return path
+
+    i = path.find('/../')
+    if i <= 0:
+      if path.startswith('/'):
+        return path[1:]
+      return path
+
+    lhs = path[0:i]
+    rhs = path[i + 4:]
+
+    i = lhs.rfind('/')
+    if i > 0:
+      path = lhs[0:i + 1] + rhs
+    else:
+      path = rhs
+
+class _File(object):
+  def __init__(self, mode, name, mark):
+    self.mode = mode
+    self.name = name
+    self.mark = mark
+
+
+class _PathMap(object):
+  def __init__(self, imp, old, new):
+    self._imp = imp
+    self._old = old
+    self._new = new
+
+  def _r(self, p):
+    return p.replace('%version%', self._imp.version)
+
+  @property
+  def old(self):
+    return self._r(self._old)
+
+  @property
+  def new(self):
+    return self._r(self._new)
+
+  def Matches(self, name):
+    return name.startswith(self.old)
+
+  def Apply(self, name):
+    return self.new + name[len(self.old):]