codekingpro/portable-devtools
114k
1# -*- coding: utf-8 -*-2#3# Copyright (C) 2013-2023 Vinay Sajip.4# Licensed to the Python Software Foundation under a contributor agreement.5# See LICENSE.txt and CONTRIBUTORS.txt.6#7from __future__ import unicode_literals8 9import base6410import codecs11import datetime12from email import message_from_file13import hashlib14import json15import logging16import os17import posixpath18import re19import shutil20import sys21import tempfile22import zipfile23 24from . import __version__, DistlibException25from .compat import sysconfig, ZipFile, fsdecode, text_type, filter26from .database import InstalledDistribution27from .metadata import Metadata, WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME28from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache,29 cached_property, get_cache_base, read_exports, tempdir,30 get_platform)31from .version import NormalizedVersion, UnsupportedVersionError32 33logger = logging.getLogger(__name__)34 35cache = None # created when needed36 37if hasattr(sys, 'pypy_version_info'): # pragma: no cover38 IMP_PREFIX = 'pp'39elif sys.platform.startswith('java'): # pragma: no cover40 IMP_PREFIX = 'jy'41elif sys.platform == 'cli': # pragma: no cover42 IMP_PREFIX = 'ip'43else:44 IMP_PREFIX = 'cp'45 46VER_SUFFIX = sysconfig.get_config_var('py_version_nodot')47if not VER_SUFFIX: # pragma: no cover48 VER_SUFFIX = '%s%s' % sys.version_info[:2]49PYVER = 'py' + VER_SUFFIX50IMPVER = IMP_PREFIX + VER_SUFFIX51 52ARCH = get_platform().replace('-', '_').replace('.', '_')53 54ABI = sysconfig.get_config_var('SOABI')55if ABI and ABI.startswith('cpython-'):56 ABI = ABI.replace('cpython-', 'cp').split('-')[0]57else:58 59 def _derive_abi():60 parts = ['cp', VER_SUFFIX]61 if sysconfig.get_config_var('Py_DEBUG'):62 parts.append('d')63 if IMP_PREFIX == 'cp':64 vi = sys.version_info[:2]65 if vi < (3, 8):66 wpm = sysconfig.get_config_var('WITH_PYMALLOC')67 if wpm is None:68 wpm = True69 if wpm:70 parts.append('m')71 if vi < (3, 3):72 us = sysconfig.get_config_var('Py_UNICODE_SIZE')73 if us == 4 or (us is None and sys.maxunicode == 0x10FFFF):74 parts.append('u')75 return ''.join(parts)76 77 ABI = _derive_abi()78 del _derive_abi79 80FILENAME_RE = re.compile(81 r'''82(?P<nm>[^-]+)83-(?P<vn>\d+[^-]*)84(-(?P<bn>\d+[^-]*))?85-(?P<py>\w+\d+(\.\w+\d+)*)86-(?P<bi>\w+)87-(?P<ar>\w+(\.\w+)*)88\.whl$89''', re.IGNORECASE | re.VERBOSE)90 91NAME_VERSION_RE = re.compile(92 r'''93(?P<nm>[^-]+)94-(?P<vn>\d+[^-]*)95(-(?P<bn>\d+[^-]*))?$96''', re.IGNORECASE | re.VERBOSE)97 98SHEBANG_RE = re.compile(br'\s*#![^\r\n]*')99SHEBANG_DETAIL_RE = re.compile(br'^(\s*#!("[^"]+"|\S+))\s+(.*)$')100SHEBANG_PYTHON = b'#!python'101SHEBANG_PYTHONW = b'#!pythonw'102 103if os.sep == '/':104 to_posix = lambda o: o105else:106 to_posix = lambda o: o.replace(os.sep, '/')107 108if sys.version_info[0] < 3:109 import imp110else:111 imp = None112 import importlib.machinery113 import importlib.util114 115 116def _get_suffixes():117 if imp:118 return [s[0] for s in imp.get_suffixes()]119 else:120 return importlib.machinery.EXTENSION_SUFFIXES121 122 123def _load_dynamic(name, path):124 # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly125 if imp:126 return imp.load_dynamic(name, path)127 else:128 spec = importlib.util.spec_from_file_location(name, path)129 module = importlib.util.module_from_spec(spec)130 sys.modules[name] = module131 spec.loader.exec_module(module)132 return module133 134 135class Mounter(object):136 137 def __init__(self):138 self.impure_wheels = {}139 self.libs = {}140 141 def add(self, pathname, extensions):142 self.impure_wheels[pathname] = extensions143 self.libs.update(extensions)144 145 def remove(self, pathname):146 extensions = self.impure_wheels.pop(pathname)147 for k, v in extensions:148 if k in self.libs:149 del self.libs[k]150 151 def find_module(self, fullname, path=None):152 if fullname in self.libs:153 result = self154 else:155 result = None156 return result157 158 def load_module(self, fullname):159 if fullname in sys.modules:160 result = sys.modules[fullname]161 else:162 if fullname not in self.libs:163 raise ImportError('unable to find extension for %s' % fullname)164 result = _load_dynamic(fullname, self.libs[fullname])165 result.__loader__ = self166 parts = fullname.rsplit('.', 1)167 if len(parts) > 1:168 result.__package__ = parts[0]169 return result170 171 172_hook = Mounter()173 174 175class Wheel(object):176 """177 Class to build and install from Wheel files (PEP 427).178 """179 180 wheel_version = (1, 1)181 hash_kind = 'sha256'182 183 def __init__(self, filename=None, sign=False, verify=False):184 """185 Initialise an instance using a (valid) filename.186 """187 self.sign = sign188 self.should_verify = verify189 self.buildver = ''190 self.pyver = [PYVER]191 self.abi = ['none']192 self.arch = ['any']193 self.dirname = os.getcwd()194 if filename is None:195 self.name = 'dummy'196 self.version = '0.1'197 self._filename = self.filename198 else:199 m = NAME_VERSION_RE.match(filename)200 if m:201 info = m.groupdict('')202 self.name = info['nm']203 # Reinstate the local version separator204 self.version = info['vn'].replace('_', '-')205 self.buildver = info['bn']206 self._filename = self.filename207 else:208 dirname, filename = os.path.split(filename)209 m = FILENAME_RE.match(filename)210 if not m:211 raise DistlibException('Invalid name or '212 'filename: %r' % filename)213 if dirname:214 self.dirname = os.path.abspath(dirname)215 self._filename = filename216 info = m.groupdict('')217 self.name = info['nm']218 self.version = info['vn']219 self.buildver = info['bn']220 self.pyver = info['py'].split('.')221 self.abi = info['bi'].split('.')222 self.arch = info['ar'].split('.')223 224 @property225 def filename(self):226 """227 Build and return a filename from the various components.228 """229 if self.buildver:230 buildver = '-' + self.buildver231 else:232 buildver = ''233 pyver = '.'.join(self.pyver)234 abi = '.'.join(self.abi)235 arch = '.'.join(self.arch)236 # replace - with _ as a local version separator237 version = self.version.replace('-', '_')238 return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver, pyver,239 abi, arch)240 241 @property242 def exists(self):243 path = os.path.join(self.dirname, self.filename)244 return os.path.isfile(path)245 246 @property247 def tags(self):248 for pyver in self.pyver:249 for abi in self.abi:250 for arch in self.arch:251 yield pyver, abi, arch252 253 @cached_property254 def metadata(self):255 pathname = os.path.join(self.dirname, self.filename)256 name_ver = '%s-%s' % (self.name, self.version)257 info_dir = '%s.dist-info' % name_ver258 wrapper = codecs.getreader('utf-8')259 with ZipFile(pathname, 'r') as zf:260 self.get_wheel_metadata(zf)261 # wv = wheel_metadata['Wheel-Version'].split('.', 1)262 # file_version = tuple([int(i) for i in wv])263 # if file_version < (1, 1):264 # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME,265 # LEGACY_METADATA_FILENAME]266 # else:267 # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME]268 fns = [WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME]269 result = None270 for fn in fns:271 try:272 metadata_filename = posixpath.join(info_dir, fn)273 with zf.open(metadata_filename) as bf:274 wf = wrapper(bf)275 result = Metadata(fileobj=wf)276 if result:277 break278 except KeyError:279 pass280 if not result:281 raise ValueError('Invalid wheel, because metadata is '282 'missing: looked in %s' % ', '.join(fns))283 return result284 285 def get_wheel_metadata(self, zf):286 name_ver = '%s-%s' % (self.name, self.version)287 info_dir = '%s.dist-info' % name_ver288 metadata_filename = posixpath.join(info_dir, 'WHEEL')289 with zf.open(metadata_filename) as bf:290 wf = codecs.getreader('utf-8')(bf)291 message = message_from_file(wf)292 return dict(message)293 294 @cached_property295 def info(self):296 pathname = os.path.join(self.dirname, self.filename)297 with ZipFile(pathname, 'r') as zf:298 result = self.get_wheel_metadata(zf)299 return result300 301 def process_shebang(self, data):302 m = SHEBANG_RE.match(data)303 if m:304 end = m.end()305 shebang, data_after_shebang = data[:end], data[end:]306 # Preserve any arguments after the interpreter307 if b'pythonw' in shebang.lower():308 shebang_python = SHEBANG_PYTHONW309 else:310 shebang_python = SHEBANG_PYTHON311 m = SHEBANG_DETAIL_RE.match(shebang)312 if m:313 args = b' ' + m.groups()[-1]314 else:315 args = b''316 shebang = shebang_python + args317 data = shebang + data_after_shebang318 else:319 cr = data.find(b'\r')320 lf = data.find(b'\n')321 if cr < 0 or cr > lf:322 term = b'\n'323 else:324 if data[cr:cr + 2] == b'\r\n':325 term = b'\r\n'326 else:327 term = b'\r'328 data = SHEBANG_PYTHON + term + data329 return data330 331 def get_hash(self, data, hash_kind=None):332 if hash_kind is None:333 hash_kind = self.hash_kind334 try:335 hasher = getattr(hashlib, hash_kind)336 except AttributeError:337 raise DistlibException('Unsupported hash algorithm: %r' %338 hash_kind)339 result = hasher(data).digest()340 result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii')341 return hash_kind, result342 343 def write_record(self, records, record_path, archive_record_path):344 records = list(records) # make a copy, as mutated345 records.append((archive_record_path, '', ''))346 with CSVWriter(record_path) as writer:347 for row in records:348 writer.writerow(row)349 350 def write_records(self, info, libdir, archive_paths):351 records = []352 distinfo, info_dir = info353 # hasher = getattr(hashlib, self.hash_kind)354 for ap, p in archive_paths:355 with open(p, 'rb') as f:356 data = f.read()357 digest = '%s=%s' % self.get_hash(data)358 size = os.path.getsize(p)359 records.append((ap, digest, size))360 361 p = os.path.join(distinfo, 'RECORD')362 ap = to_posix(os.path.join(info_dir, 'RECORD'))363 self.write_record(records, p, ap)364 archive_paths.append((ap, p))365 366 def build_zip(self, pathname, archive_paths):367 with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf:368 for ap, p in archive_paths:369 logger.debug('Wrote %s to %s in wheel', p, ap)370 zf.write(p, ap)371 372 def build(self, paths, tags=None, wheel_version=None):373 """374 Build a wheel from files in specified paths, and use any specified tags375 when determining the name of the wheel.376 """377 if tags is None:378 tags = {}379 380 libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0]381 if libkey == 'platlib':382 is_pure = 'false'383 default_pyver = [IMPVER]384 default_abi = [ABI]385 default_arch = [ARCH]386 else:387 is_pure = 'true'388 default_pyver = [PYVER]389 default_abi = ['none']390 default_arch = ['any']391 392 self.pyver = tags.get('pyver', default_pyver)393 self.abi = tags.get('abi', default_abi)394 self.arch = tags.get('arch', default_arch)395 396 libdir = paths[libkey]397 398 name_ver = '%s-%s' % (self.name, self.version)399 data_dir = '%s.data' % name_ver400 info_dir = '%s.dist-info' % name_ver401 402 archive_paths = []403 404 # First, stuff which is not in site-packages405 for key in ('data', 'headers', 'scripts'):406 if key not in paths:407 continue408 path = paths[key]409 if os.path.isdir(path):410 for root, dirs, files in os.walk(path):411 for fn in files:412 p = fsdecode(os.path.join(root, fn))413 rp = os.path.relpath(p, path)414 ap = to_posix(os.path.join(data_dir, key, rp))415 archive_paths.append((ap, p))416 if key == 'scripts' and not p.endswith('.exe'):417 with open(p, 'rb') as f:418 data = f.read()419 data = self.process_shebang(data)420 with open(p, 'wb') as f:421 f.write(data)422 423 # Now, stuff which is in site-packages, other than the424 # distinfo stuff.425 path = libdir426 distinfo = None427 for root, dirs, files in os.walk(path):428 if root == path:429 # At the top level only, save distinfo for later430 # and skip it for now431 for i, dn in enumerate(dirs):432 dn = fsdecode(dn)433 if dn.endswith('.dist-info'):434 distinfo = os.path.join(root, dn)435 del dirs[i]436 break437 assert distinfo, '.dist-info directory expected, not found'438 439 for fn in files:440 # comment out next suite to leave .pyc files in441 if fsdecode(fn).endswith(('.pyc', '.pyo')):442 continue443 p = os.path.join(root, fn)444 rp = to_posix(os.path.relpath(p, path))445 archive_paths.append((rp, p))446 447 # Now distinfo. Assumed to be flat, i.e. os.listdir is enough.448 files = os.listdir(distinfo)449 for fn in files:450 if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'):451 p = fsdecode(os.path.join(distinfo, fn))452 ap = to_posix(os.path.join(info_dir, fn))453 archive_paths.append((ap, p))454 455 wheel_metadata = [456 'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version),457 'Generator: distlib %s' % __version__,458 'Root-Is-Purelib: %s' % is_pure,459 ]460 for pyver, abi, arch in self.tags:461 wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch))462 p = os.path.join(distinfo, 'WHEEL')463 with open(p, 'w') as f:464 f.write('\n'.join(wheel_metadata))465 ap = to_posix(os.path.join(info_dir, 'WHEEL'))466 archive_paths.append((ap, p))467 468 # sort the entries by archive path. Not needed by any spec, but it469 # keeps the archive listing and RECORD tidier than they would otherwise470 # be. Use the number of path segments to keep directory entries together,471 # and keep the dist-info stuff at the end.472 def sorter(t):473 ap = t[0]474 n = ap.count('/')475 if '.dist-info' in ap:476 n += 10000477 return (n, ap)478 479 archive_paths = sorted(archive_paths, key=sorter)480 481 # Now, at last, RECORD.482 # Paths in here are archive paths - nothing else makes sense.483 self.write_records((distinfo, info_dir), libdir, archive_paths)484 # Now, ready to build the zip file485 pathname = os.path.join(self.dirname, self.filename)486 self.build_zip(pathname, archive_paths)487 return pathname488 489 def skip_entry(self, arcname):490 """491 Determine whether an archive entry should be skipped when verifying492 or installing.493 """494 # The signature file won't be in RECORD,495 # and we don't currently don't do anything with it496 # We also skip directories, as they won't be in RECORD497 # either. See:498 #499 # https://github.com/pypa/wheel/issues/294500 # https://github.com/pypa/wheel/issues/287501 # https://github.com/pypa/wheel/pull/289502 #503 return arcname.endswith(('/', '/RECORD.jws'))504 505 def install(self, paths, maker, **kwargs):506 """507 Install a wheel to the specified paths. If kwarg ``warner`` is508 specified, it should be a callable, which will be called with two509 tuples indicating the wheel version of this software and the wheel510 version in the file, if there is a discrepancy in the versions.511 This can be used to issue any warnings to raise any exceptions.512 If kwarg ``lib_only`` is True, only the purelib/platlib files are513 installed, and the headers, scripts, data and dist-info metadata are514 not written. If kwarg ``bytecode_hashed_invalidation`` is True, written515 bytecode will try to use file-hash based invalidation (PEP-552) on516 supported interpreter versions (CPython 2.7+).517 518 The return value is a :class:`InstalledDistribution` instance unless519 ``options.lib_only`` is True, in which case the return value is ``None``.520 """521 522 dry_run = maker.dry_run523 warner = kwargs.get('warner')524 lib_only = kwargs.get('lib_only', False)525 bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation',526 False)527 528 pathname = os.path.join(self.dirname, self.filename)529 name_ver = '%s-%s' % (self.name, self.version)530 data_dir = '%s.data' % name_ver531 info_dir = '%s.dist-info' % name_ver532 533 metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)534 wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')535 record_name = posixpath.join(info_dir, 'RECORD')536 537 wrapper = codecs.getreader('utf-8')538 539 with ZipFile(pathname, 'r') as zf:540 with zf.open(wheel_metadata_name) as bwf:541 wf = wrapper(bwf)542 message = message_from_file(wf)543 wv = message['Wheel-Version'].split('.', 1)544 file_version = tuple([int(i) for i in wv])545 if (file_version != self.wheel_version) and warner:546 warner(self.wheel_version, file_version)547 548 if message['Root-Is-Purelib'] == 'true':549 libdir = paths['purelib']550 else:551 libdir = paths['platlib']552 553 records = {}554 with zf.open(record_name) as bf:555 with CSVReader(stream=bf) as reader:556 for row in reader:557 p = row[0]558 records[p] = row559 560 data_pfx = posixpath.join(data_dir, '')561 info_pfx = posixpath.join(info_dir, '')562 script_pfx = posixpath.join(data_dir, 'scripts', '')563 564 # make a new instance rather than a copy of maker's,565 # as we mutate it566 fileop = FileOperator(dry_run=dry_run)567 fileop.record = True # so we can rollback if needed568 569 bc = not sys.dont_write_bytecode # Double negatives. Lovely!570 571 outfiles = [] # for RECORD writing572 573 # for script copying/shebang processing574 workdir = tempfile.mkdtemp()575 # set target dir later576 # we default add_launchers to False, as the577 # Python Launcher should be used instead578 maker.source_dir = workdir579 maker.target_dir = None580 try:581 for zinfo in zf.infolist():582 arcname = zinfo.filename583 if isinstance(arcname, text_type):584 u_arcname = arcname585 else:586 u_arcname = arcname.decode('utf-8')587 if self.skip_entry(u_arcname):588 continue589 row = records[u_arcname]590 if row[2] and str(zinfo.file_size) != row[2]:591 raise DistlibException('size mismatch for '592 '%s' % u_arcname)593 if row[1]:594 kind, value = row[1].split('=', 1)595 with zf.open(arcname) as bf:596 data = bf.read()597 _, digest = self.get_hash(data, kind)598 if digest != value:599 raise DistlibException('digest mismatch for '600 '%s' % arcname)601 602 if lib_only and u_arcname.startswith((info_pfx, data_pfx)):603 logger.debug('lib_only: skipping %s', u_arcname)604 continue605 is_script = (u_arcname.startswith(script_pfx)606 and not u_arcname.endswith('.exe'))607 608 if u_arcname.startswith(data_pfx):609 _, where, rp = u_arcname.split('/', 2)610 outfile = os.path.join(paths[where], convert_path(rp))611 else:612 # meant for site-packages.613 if u_arcname in (wheel_metadata_name, record_name):614 continue615 outfile = os.path.join(libdir, convert_path(u_arcname))616 if not is_script:617 with zf.open(arcname) as bf:618 fileop.copy_stream(bf, outfile)619 # Issue #147: permission bits aren't preserved. Using620 # zf.extract(zinfo, libdir) should have worked, but didn't,621 # see https://www.thetopsites.net/article/53834422.shtml622 # So ... manually preserve permission bits as given in zinfo623 if os.name == 'posix':624 # just set the normal permission bits625 os.chmod(outfile,626 (zinfo.external_attr >> 16) & 0x1FF)627 outfiles.append(outfile)628 # Double check the digest of the written file629 if not dry_run and row[1]:630 with open(outfile, 'rb') as bf:631 data = bf.read()632 _, newdigest = self.get_hash(data, kind)633 if newdigest != digest:634 raise DistlibException('digest mismatch '635 'on write for '636 '%s' % outfile)637 if bc and outfile.endswith('.py'):638 try:639 pyc = fileop.byte_compile(640 outfile,641 hashed_invalidation=bc_hashed_invalidation)642 outfiles.append(pyc)643 except Exception:644 # Don't give up if byte-compilation fails,645 # but log it and perhaps warn the user646 logger.warning('Byte-compilation failed',647 exc_info=True)648 else:649 fn = os.path.basename(convert_path(arcname))650 workname = os.path.join(workdir, fn)651 with zf.open(arcname) as bf:652 fileop.copy_stream(bf, workname)653 654 dn, fn = os.path.split(outfile)655 maker.target_dir = dn656 filenames = maker.make(fn)657 fileop.set_executable_mode(filenames)658 outfiles.extend(filenames)659 660 if lib_only:661 logger.debug('lib_only: returning None')662 dist = None663 else:664 # Generate scripts665 666 # Try to get pydist.json so we can see if there are667 # any commands to generate. If this fails (e.g. because668 # of a legacy wheel), log a warning but don't give up.669 commands = None670 file_version = self.info['Wheel-Version']671 if file_version == '1.0':672 # Use legacy info673 ep = posixpath.join(info_dir, 'entry_points.txt')674 try:675 with zf.open(ep) as bwf:676 epdata = read_exports(bwf)677 commands = {}678 for key in ('console', 'gui'):679 k = '%s_scripts' % key680 if k in epdata:681 commands['wrap_%s' % key] = d = {}682 for v in epdata[k].values():683 s = '%s:%s' % (v.prefix, v.suffix)684 if v.flags:685 s += ' [%s]' % ','.join(v.flags)686 d[v.name] = s687 except Exception:688 logger.warning('Unable to read legacy script '689 'metadata, so cannot generate '690 'scripts')691 else:692 try:693 with zf.open(metadata_name) as bwf:694 wf = wrapper(bwf)695 commands = json.load(wf).get('extensions')696 if commands:697 commands = commands.get('python.commands')698 except Exception:699 logger.warning('Unable to read JSON metadata, so '700 'cannot generate scripts')701 if commands:702 console_scripts = commands.get('wrap_console', {})703 gui_scripts = commands.get('wrap_gui', {})704 if console_scripts or gui_scripts:705 script_dir = paths.get('scripts', '')706 if not os.path.isdir(script_dir):707 raise ValueError('Valid script path not '708 'specified')709 maker.target_dir = script_dir710 for k, v in console_scripts.items():711 script = '%s = %s' % (k, v)712 filenames = maker.make(script)713 fileop.set_executable_mode(filenames)714 715 if gui_scripts:716 options = {'gui': True}717 for k, v in gui_scripts.items():718 script = '%s = %s' % (k, v)719 filenames = maker.make(script, options)720 fileop.set_executable_mode(filenames)721 722 p = os.path.join(libdir, info_dir)723 dist = InstalledDistribution(p)724 725 # Write SHARED726 paths = dict(paths) # don't change passed in dict727 del paths['purelib']728 del paths['platlib']729 paths['lib'] = libdir730 p = dist.write_shared_locations(paths, dry_run)731 if p:732 outfiles.append(p)733 734 # Write RECORD735 dist.write_installed_files(outfiles, paths['prefix'],736 dry_run)737 return dist738 except Exception: # pragma: no cover739 logger.exception('installation failed.')740 fileop.rollback()741 raise742 finally:743 shutil.rmtree(workdir)744 745 def _get_dylib_cache(self):746 global cache747 if cache is None:748 # Use native string to avoid issues on 2.x: see Python #20140.749 base = os.path.join(get_cache_base(), str('dylib-cache'),750 '%s.%s' % sys.version_info[:2])751 cache = Cache(base)752 return cache753 754 def _get_extensions(self):755 pathname = os.path.join(self.dirname, self.filename)756 name_ver = '%s-%s' % (self.name, self.version)757 info_dir = '%s.dist-info' % name_ver758 arcname = posixpath.join(info_dir, 'EXTENSIONS')759 wrapper = codecs.getreader('utf-8')760 result = []761 with ZipFile(pathname, 'r') as zf:762 try:763 with zf.open(arcname) as bf:764 wf = wrapper(bf)765 extensions = json.load(wf)766 cache = self._get_dylib_cache()767 prefix = cache.prefix_to_dir(pathname)768 cache_base = os.path.join(cache.base, prefix)769 if not os.path.isdir(cache_base):770 os.makedirs(cache_base)771 for name, relpath in extensions.items():772 dest = os.path.join(cache_base, convert_path(relpath))773 if not os.path.exists(dest):774 extract = True775 else:776 file_time = os.stat(dest).st_mtime777 file_time = datetime.datetime.fromtimestamp(778 file_time)779 info = zf.getinfo(relpath)780 wheel_time = datetime.datetime(*info.date_time)781 extract = wheel_time > file_time782 if extract:783 zf.extract(relpath, cache_base)784 result.append((name, dest))785 except KeyError:786 pass787 return result788 789 def is_compatible(self):790 """791 Determine if a wheel is compatible with the running system.792 """793 return is_compatible(self)794 795 def is_mountable(self):796 """797 Determine if a wheel is asserted as mountable by its metadata.798 """799 return True # for now - metadata details TBD800 801 def mount(self, append=False):802 pathname = os.path.abspath(os.path.join(self.dirname, self.filename))803 if not self.is_compatible():804 msg = 'Wheel %s not compatible with this Python.' % pathname805 raise DistlibException(msg)806 if not self.is_mountable():807 msg = 'Wheel %s is marked as not mountable.' % pathname808 raise DistlibException(msg)809 if pathname in sys.path:810 logger.debug('%s already in path', pathname)811 else:812 if append:813 sys.path.append(pathname)814 else:815 sys.path.insert(0, pathname)816 extensions = self._get_extensions()817 if extensions:818 if _hook not in sys.meta_path:819 sys.meta_path.append(_hook)820 _hook.add(pathname, extensions)821 822 def unmount(self):823 pathname = os.path.abspath(os.path.join(self.dirname, self.filename))824 if pathname not in sys.path:825 logger.debug('%s not in path', pathname)826 else:827 sys.path.remove(pathname)828 if pathname in _hook.impure_wheels:829 _hook.remove(pathname)830 if not _hook.impure_wheels:831 if _hook in sys.meta_path:832 sys.meta_path.remove(_hook)833 834 def verify(self):835 pathname = os.path.join(self.dirname, self.filename)836 name_ver = '%s-%s' % (self.name, self.version)837 # data_dir = '%s.data' % name_ver838 info_dir = '%s.dist-info' % name_ver839 840 # metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)841 wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')842 record_name = posixpath.join(info_dir, 'RECORD')843 844 wrapper = codecs.getreader('utf-8')845 846 with ZipFile(pathname, 'r') as zf:847 with zf.open(wheel_metadata_name) as bwf:848 wf = wrapper(bwf)849 message_from_file(wf)850 # wv = message['Wheel-Version'].split('.', 1)851 # file_version = tuple([int(i) for i in wv])852 # TODO version verification853 854 records = {}855 with zf.open(record_name) as bf:856 with CSVReader(stream=bf) as reader:857 for row in reader:858 p = row[0]859 records[p] = row860 861 for zinfo in zf.infolist():862 arcname = zinfo.filename863 if isinstance(arcname, text_type):864 u_arcname = arcname865 else:866 u_arcname = arcname.decode('utf-8')867 # See issue #115: some wheels have .. in their entries, but868 # in the filename ... e.g. __main__..py ! So the check is869 # updated to look for .. in the directory portions870 p = u_arcname.split('/')871 if '..' in p:872 raise DistlibException('invalid entry in '873 'wheel: %r' % u_arcname)874 875 if self.skip_entry(u_arcname):876 continue877 row = records[u_arcname]878 if row[2] and str(zinfo.file_size) != row[2]:879 raise DistlibException('size mismatch for '880 '%s' % u_arcname)881 if row[1]:882 kind, value = row[1].split('=', 1)883 with zf.open(arcname) as bf:884 data = bf.read()885 _, digest = self.get_hash(data, kind)886 if digest != value:887 raise DistlibException('digest mismatch for '888 '%s' % arcname)889 890 def update(self, modifier, dest_dir=None, **kwargs):891 """892 Update the contents of a wheel in a generic way. The modifier should893 be a callable which expects a dictionary argument: its keys are894 archive-entry paths, and its values are absolute filesystem paths895 where the contents the corresponding archive entries can be found. The896 modifier is free to change the contents of the files pointed to, add897 new entries and remove entries, before returning. This method will898 extract the entire contents of the wheel to a temporary location, call899 the modifier, and then use the passed (and possibly updated)900 dictionary to write a new wheel. If ``dest_dir`` is specified, the new901 wheel is written there -- otherwise, the original wheel is overwritten.902 903 The modifier should return True if it updated the wheel, else False.904 This method returns the same value the modifier returns.905 """906 907 def get_version(path_map, info_dir):908 version = path = None909 key = '%s/%s' % (info_dir, LEGACY_METADATA_FILENAME)910 if key not in path_map:911 key = '%s/PKG-INFO' % info_dir912 if key in path_map:913 path = path_map[key]914 version = Metadata(path=path).version915 return version, path916 917 def update_version(version, path):918 updated = None919 try:920 NormalizedVersion(version)921 i = version.find('-')922 if i < 0:923 updated = '%s+1' % version924 else:925 parts = [int(s) for s in version[i + 1:].split('.')]926 parts[-1] += 1927 updated = '%s+%s' % (version[:i], '.'.join(928 str(i) for i in parts))929 except UnsupportedVersionError:930 logger.debug(931 'Cannot update non-compliant (PEP-440) '932 'version %r', version)933 if updated:934 md = Metadata(path=path)935 md.version = updated936 legacy = path.endswith(LEGACY_METADATA_FILENAME)937 md.write(path=path, legacy=legacy)938 logger.debug('Version updated from %r to %r', version, updated)939 940 pathname = os.path.join(self.dirname, self.filename)941 name_ver = '%s-%s' % (self.name, self.version)942 info_dir = '%s.dist-info' % name_ver943 record_name = posixpath.join(info_dir, 'RECORD')944 with tempdir() as workdir:945 with ZipFile(pathname, 'r') as zf:946 path_map = {}947 for zinfo in zf.infolist():948 arcname = zinfo.filename949 if isinstance(arcname, text_type):950 u_arcname = arcname951 else:952 u_arcname = arcname.decode('utf-8')953 if u_arcname == record_name:954 continue955 if '..' in u_arcname:956 raise DistlibException('invalid entry in '957 'wheel: %r' % u_arcname)958 zf.extract(zinfo, workdir)959 path = os.path.join(workdir, convert_path(u_arcname))960 path_map[u_arcname] = path961 962 # Remember the version.963 original_version, _ = get_version(path_map, info_dir)964 # Files extracted. Call the modifier.965 modified = modifier(path_map, **kwargs)966 if modified:967 # Something changed - need to build a new wheel.968 current_version, path = get_version(path_map, info_dir)969 if current_version and (current_version == original_version):970 # Add or update local version to signify changes.971 update_version(current_version, path)972 # Decide where the new wheel goes.973 if dest_dir is None:974 fd, newpath = tempfile.mkstemp(suffix='.whl',975 prefix='wheel-update-',976 dir=workdir)977 os.close(fd)978 else:979 if not os.path.isdir(dest_dir):980 raise DistlibException('Not a directory: %r' %981 dest_dir)982 newpath = os.path.join(dest_dir, self.filename)983 archive_paths = list(path_map.items())984 distinfo = os.path.join(workdir, info_dir)985 info = distinfo, info_dir986 self.write_records(info, workdir, archive_paths)987 self.build_zip(newpath, archive_paths)988 if dest_dir is None:989 shutil.copyfile(newpath, pathname)990 return modified991 992 993def _get_glibc_version():994 import platform995 ver = platform.libc_ver()996 result = []997 if ver[0] == 'glibc':998 for s in ver[1].split('.'):999 result.append(int(s) if s.isdigit() else 0)1000 result = tuple(result)1001 return result1002 1003 1004def compatible_tags():1005 """1006 Return (pyver, abi, arch) tuples compatible with this Python.1007 """1008 versions = [VER_SUFFIX]1009 major = VER_SUFFIX[0]1010 for minor in range(sys.version_info[1] - 1, -1, -1):1011 versions.append(''.join([major, str(minor)]))1012 1013 abis = []1014 for suffix in _get_suffixes():1015 if suffix.startswith('.abi'):1016 abis.append(suffix.split('.', 2)[1])1017 abis.sort()1018 if ABI != 'none':1019 abis.insert(0, ABI)1020 abis.append('none')1021 result = []1022 1023 arches = [ARCH]1024 if sys.platform == 'darwin':1025 m = re.match(r'(\w+)_(\d+)_(\d+)_(\w+)$', ARCH)1026 if m:1027 name, major, minor, arch = m.groups()1028 minor = int(minor)1029 matches = [arch]1030 if arch in ('i386', 'ppc'):1031 matches.append('fat')1032 if arch in ('i386', 'ppc', 'x86_64'):1033 matches.append('fat3')1034 if arch in ('ppc64', 'x86_64'):1035 matches.append('fat64')1036 if arch in ('i386', 'x86_64'):1037 matches.append('intel')1038 if arch in ('i386', 'x86_64', 'intel', 'ppc', 'ppc64'):1039 matches.append('universal')1040 while minor >= 0:1041 for match in matches:1042 s = '%s_%s_%s_%s' % (name, major, minor, match)1043 if s != ARCH: # already there1044 arches.append(s)1045 minor -= 11046 1047 # Most specific - our Python version, ABI and arch1048 for abi in abis:1049 for arch in arches:1050 result.append((''.join((IMP_PREFIX, versions[0])), abi, arch))1051 # manylinux1052 if abi != 'none' and sys.platform.startswith('linux'):1053 arch = arch.replace('linux_', '')1054 parts = _get_glibc_version()1055 if len(parts) == 2:1056 if parts >= (2, 5):1057 result.append((''.join((IMP_PREFIX, versions[0])), abi,1058 'manylinux1_%s' % arch))1059 if parts >= (2, 12):1060 result.append((''.join((IMP_PREFIX, versions[0])), abi,1061 'manylinux2010_%s' % arch))1062 if parts >= (2, 17):1063 result.append((''.join((IMP_PREFIX, versions[0])), abi,1064 'manylinux2014_%s' % arch))1065 result.append(1066 (''.join((IMP_PREFIX, versions[0])), abi,1067 'manylinux_%s_%s_%s' % (parts[0], parts[1], arch)))1068 1069 # where no ABI / arch dependency, but IMP_PREFIX dependency1070 for i, version in enumerate(versions):1071 result.append((''.join((IMP_PREFIX, version)), 'none', 'any'))1072 if i == 0:1073 result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any'))1074 1075 # no IMP_PREFIX, ABI or arch dependency1076 for i, version in enumerate(versions):1077 result.append((''.join(('py', version)), 'none', 'any'))1078 if i == 0:1079 result.append((''.join(('py', version[0])), 'none', 'any'))1080 1081 return set(result)1082 1083 1084COMPATIBLE_TAGS = compatible_tags()1085 1086del compatible_tags1087 1088 1089def is_compatible(wheel, tags=None):1090 if not isinstance(wheel, Wheel):1091 wheel = Wheel(wheel) # assume it's a filename1092 result = False1093 if tags is None:1094 tags = COMPATIBLE_TAGS1095 for ver, abi, arch in tags:1096 if ver in wheel.pyver and abi in wheel.abi and arch in wheel.arch:1097 result = True1098 break1099 return result1100