codekingpro/portable-devtools
114k
1# -*- coding: utf-8 -*-2#3# Copyright (C) 2012-2023 The Python Software Foundation.4# See LICENSE.txt and CONTRIBUTORS.txt.5#6"""PEP 376 implementation."""7 8from __future__ import unicode_literals9 10import base6411import codecs12import contextlib13import hashlib14import logging15import os16import posixpath17import sys18import zipimport19 20from . import DistlibException, resources21from .compat import StringIO22from .version import get_scheme, UnsupportedVersionError23from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME,24 LEGACY_METADATA_FILENAME)25from .util import (parse_requirement, cached_property, parse_name_and_version,26 read_exports, write_exports, CSVReader, CSVWriter)27 28__all__ = [29 'Distribution', 'BaseInstalledDistribution', 'InstalledDistribution',30 'EggInfoDistribution', 'DistributionPath'31]32 33logger = logging.getLogger(__name__)34 35EXPORTS_FILENAME = 'pydist-exports.json'36COMMANDS_FILENAME = 'pydist-commands.json'37 38DIST_FILES = ('INSTALLER', METADATA_FILENAME, 'RECORD', 'REQUESTED',39 'RESOURCES', EXPORTS_FILENAME, 'SHARED')40 41DISTINFO_EXT = '.dist-info'42 43 44class _Cache(object):45 """46 A simple cache mapping names and .dist-info paths to distributions47 """48 49 def __init__(self):50 """51 Initialise an instance. There is normally one for each DistributionPath.52 """53 self.name = {}54 self.path = {}55 self.generated = False56 57 def clear(self):58 """59 Clear the cache, setting it to its initial state.60 """61 self.name.clear()62 self.path.clear()63 self.generated = False64 65 def add(self, dist):66 """67 Add a distribution to the cache.68 :param dist: The distribution to add.69 """70 if dist.path not in self.path:71 self.path[dist.path] = dist72 self.name.setdefault(dist.key, []).append(dist)73 74 75class DistributionPath(object):76 """77 Represents a set of distributions installed on a path (typically sys.path).78 """79 80 def __init__(self, path=None, include_egg=False):81 """82 Create an instance from a path, optionally including legacy (distutils/83 setuptools/distribute) distributions.84 :param path: The path to use, as a list of directories. If not specified,85 sys.path is used.86 :param include_egg: If True, this instance will look for and return legacy87 distributions as well as those based on PEP 376.88 """89 if path is None:90 path = sys.path91 self.path = path92 self._include_dist = True93 self._include_egg = include_egg94 95 self._cache = _Cache()96 self._cache_egg = _Cache()97 self._cache_enabled = True98 self._scheme = get_scheme('default')99 100 def _get_cache_enabled(self):101 return self._cache_enabled102 103 def _set_cache_enabled(self, value):104 self._cache_enabled = value105 106 cache_enabled = property(_get_cache_enabled, _set_cache_enabled)107 108 def clear_cache(self):109 """110 Clears the internal cache.111 """112 self._cache.clear()113 self._cache_egg.clear()114 115 def _yield_distributions(self):116 """117 Yield .dist-info and/or .egg(-info) distributions.118 """119 # We need to check if we've seen some resources already, because on120 # some Linux systems (e.g. some Debian/Ubuntu variants) there are121 # symlinks which alias other files in the environment.122 seen = set()123 for path in self.path:124 finder = resources.finder_for_path(path)125 if finder is None:126 continue127 r = finder.find('')128 if not r or not r.is_container:129 continue130 rset = sorted(r.resources)131 for entry in rset:132 r = finder.find(entry)133 if not r or r.path in seen:134 continue135 try:136 if self._include_dist and entry.endswith(DISTINFO_EXT):137 possible_filenames = [138 METADATA_FILENAME, WHEEL_METADATA_FILENAME,139 LEGACY_METADATA_FILENAME140 ]141 for metadata_filename in possible_filenames:142 metadata_path = posixpath.join(143 entry, metadata_filename)144 pydist = finder.find(metadata_path)145 if pydist:146 break147 else:148 continue149 150 with contextlib.closing(pydist.as_stream()) as stream:151 metadata = Metadata(fileobj=stream,152 scheme='legacy')153 logger.debug('Found %s', r.path)154 seen.add(r.path)155 yield new_dist_class(r.path,156 metadata=metadata,157 env=self)158 elif self._include_egg and entry.endswith(159 ('.egg-info', '.egg')):160 logger.debug('Found %s', r.path)161 seen.add(r.path)162 yield old_dist_class(r.path, self)163 except Exception as e:164 msg = 'Unable to read distribution at %s, perhaps due to bad metadata: %s'165 logger.warning(msg, r.path, e)166 import warnings167 warnings.warn(msg % (r.path, e), stacklevel=2)168 169 def _generate_cache(self):170 """171 Scan the path for distributions and populate the cache with172 those that are found.173 """174 gen_dist = not self._cache.generated175 gen_egg = self._include_egg and not self._cache_egg.generated176 if gen_dist or gen_egg:177 for dist in self._yield_distributions():178 if isinstance(dist, InstalledDistribution):179 self._cache.add(dist)180 else:181 self._cache_egg.add(dist)182 183 if gen_dist:184 self._cache.generated = True185 if gen_egg:186 self._cache_egg.generated = True187 188 @classmethod189 def distinfo_dirname(cls, name, version):190 """191 The *name* and *version* parameters are converted into their192 filename-escaped form, i.e. any ``'-'`` characters are replaced193 with ``'_'`` other than the one in ``'dist-info'`` and the one194 separating the name from the version number.195 196 :parameter name: is converted to a standard distribution name by replacing197 any runs of non- alphanumeric characters with a single198 ``'-'``.199 :type name: string200 :parameter version: is converted to a standard version string. Spaces201 become dots, and all other non-alphanumeric characters202 (except dots) become dashes, with runs of multiple203 dashes condensed to a single dash.204 :type version: string205 :returns: directory name206 :rtype: string"""207 name = name.replace('-', '_')208 return '-'.join([name, version]) + DISTINFO_EXT209 210 def get_distributions(self):211 """212 Provides an iterator that looks for distributions and returns213 :class:`InstalledDistribution` or214 :class:`EggInfoDistribution` instances for each one of them.215 216 :rtype: iterator of :class:`InstalledDistribution` and217 :class:`EggInfoDistribution` instances218 """219 if not self._cache_enabled:220 for dist in self._yield_distributions():221 yield dist222 else:223 self._generate_cache()224 225 for dist in self._cache.path.values():226 yield dist227 228 if self._include_egg:229 for dist in self._cache_egg.path.values():230 yield dist231 232 def get_distribution(self, name):233 """234 Looks for a named distribution on the path.235 236 This function only returns the first result found, as no more than one237 value is expected. If nothing is found, ``None`` is returned.238 239 :rtype: :class:`InstalledDistribution`, :class:`EggInfoDistribution`240 or ``None``241 """242 result = None243 name = name.lower()244 if not self._cache_enabled:245 for dist in self._yield_distributions():246 if dist.key == name:247 result = dist248 break249 else:250 self._generate_cache()251 252 if name in self._cache.name:253 result = self._cache.name[name][0]254 elif self._include_egg and name in self._cache_egg.name:255 result = self._cache_egg.name[name][0]256 return result257 258 def provides_distribution(self, name, version=None):259 """260 Iterates over all distributions to find which distributions provide *name*.261 If a *version* is provided, it will be used to filter the results.262 263 This function only returns the first result found, since no more than264 one values are expected. If the directory is not found, returns ``None``.265 266 :parameter version: a version specifier that indicates the version267 required, conforming to the format in ``PEP-345``268 269 :type name: string270 :type version: string271 """272 matcher = None273 if version is not None:274 try:275 matcher = self._scheme.matcher('%s (%s)' % (name, version))276 except ValueError:277 raise DistlibException('invalid name or version: %r, %r' %278 (name, version))279 280 for dist in self.get_distributions():281 # We hit a problem on Travis where enum34 was installed and doesn't282 # have a provides attribute ...283 if not hasattr(dist, 'provides'):284 logger.debug('No "provides": %s', dist)285 else:286 provided = dist.provides287 288 for p in provided:289 p_name, p_ver = parse_name_and_version(p)290 if matcher is None:291 if p_name == name:292 yield dist293 break294 else:295 if p_name == name and matcher.match(p_ver):296 yield dist297 break298 299 def get_file_path(self, name, relative_path):300 """301 Return the path to a resource file.302 """303 dist = self.get_distribution(name)304 if dist is None:305 raise LookupError('no distribution named %r found' % name)306 return dist.get_resource_path(relative_path)307 308 def get_exported_entries(self, category, name=None):309 """310 Return all of the exported entries in a particular category.311 312 :param category: The category to search for entries.313 :param name: If specified, only entries with that name are returned.314 """315 for dist in self.get_distributions():316 r = dist.exports317 if category in r:318 d = r[category]319 if name is not None:320 if name in d:321 yield d[name]322 else:323 for v in d.values():324 yield v325 326 327class Distribution(object):328 """329 A base class for distributions, whether installed or from indexes.330 Either way, it must have some metadata, so that's all that's needed331 for construction.332 """333 334 build_time_dependency = False335 """336 Set to True if it's known to be only a build-time dependency (i.e.337 not needed after installation).338 """339 340 requested = False341 """A boolean that indicates whether the ``REQUESTED`` metadata file is342 present (in other words, whether the package was installed by user343 request or it was installed as a dependency)."""344 345 def __init__(self, metadata):346 """347 Initialise an instance.348 :param metadata: The instance of :class:`Metadata` describing this349 distribution.350 """351 self.metadata = metadata352 self.name = metadata.name353 self.key = self.name.lower() # for case-insensitive comparisons354 self.version = metadata.version355 self.locator = None356 self.digest = None357 self.extras = None # additional features requested358 self.context = None # environment marker overrides359 self.download_urls = set()360 self.digests = {}361 362 @property363 def source_url(self):364 """365 The source archive download URL for this distribution.366 """367 return self.metadata.source_url368 369 download_url = source_url # Backward compatibility370 371 @property372 def name_and_version(self):373 """374 A utility property which displays the name and version in parentheses.375 """376 return '%s (%s)' % (self.name, self.version)377 378 @property379 def provides(self):380 """381 A set of distribution names and versions provided by this distribution.382 :return: A set of "name (version)" strings.383 """384 plist = self.metadata.provides385 s = '%s (%s)' % (self.name, self.version)386 if s not in plist:387 plist.append(s)388 return plist389 390 def _get_requirements(self, req_attr):391 md = self.metadata392 reqts = getattr(md, req_attr)393 logger.debug('%s: got requirements %r from metadata: %r', self.name,394 req_attr, reqts)395 return set(396 md.get_requirements(reqts, extras=self.extras, env=self.context))397 398 @property399 def run_requires(self):400 return self._get_requirements('run_requires')401 402 @property403 def meta_requires(self):404 return self._get_requirements('meta_requires')405 406 @property407 def build_requires(self):408 return self._get_requirements('build_requires')409 410 @property411 def test_requires(self):412 return self._get_requirements('test_requires')413 414 @property415 def dev_requires(self):416 return self._get_requirements('dev_requires')417 418 def matches_requirement(self, req):419 """420 Say if this instance matches (fulfills) a requirement.421 :param req: The requirement to match.422 :rtype req: str423 :return: True if it matches, else False.424 """425 # Requirement may contain extras - parse to lose those426 # from what's passed to the matcher427 r = parse_requirement(req)428 scheme = get_scheme(self.metadata.scheme)429 try:430 matcher = scheme.matcher(r.requirement)431 except UnsupportedVersionError:432 # XXX compat-mode if cannot read the version433 logger.warning('could not read version %r - using name only', req)434 name = req.split()[0]435 matcher = scheme.matcher(name)436 437 name = matcher.key # case-insensitive438 439 result = False440 for p in self.provides:441 p_name, p_ver = parse_name_and_version(p)442 if p_name != name:443 continue444 try:445 result = matcher.match(p_ver)446 break447 except UnsupportedVersionError:448 pass449 return result450 451 def __repr__(self):452 """453 Return a textual representation of this instance,454 """455 if self.source_url:456 suffix = ' [%s]' % self.source_url457 else:458 suffix = ''459 return '<Distribution %s (%s)%s>' % (self.name, self.version, suffix)460 461 def __eq__(self, other):462 """463 See if this distribution is the same as another.464 :param other: The distribution to compare with. To be equal to one465 another. distributions must have the same type, name,466 version and source_url.467 :return: True if it is the same, else False.468 """469 if type(other) is not type(self):470 result = False471 else:472 result = (self.name == other.name and self.version == other.version473 and self.source_url == other.source_url)474 return result475 476 def __hash__(self):477 """478 Compute hash in a way which matches the equality test.479 """480 return hash(self.name) + hash(self.version) + hash(self.source_url)481 482 483class BaseInstalledDistribution(Distribution):484 """485 This is the base class for installed distributions (whether PEP 376 or486 legacy).487 """488 489 hasher = None490 491 def __init__(self, metadata, path, env=None):492 """493 Initialise an instance.494 :param metadata: An instance of :class:`Metadata` which describes the495 distribution. This will normally have been initialised496 from a metadata file in the ``path``.497 :param path: The path of the ``.dist-info`` or ``.egg-info``498 directory for the distribution.499 :param env: This is normally the :class:`DistributionPath`500 instance where this distribution was found.501 """502 super(BaseInstalledDistribution, self).__init__(metadata)503 self.path = path504 self.dist_path = env505 506 def get_hash(self, data, hasher=None):507 """508 Get the hash of some data, using a particular hash algorithm, if509 specified.510 511 :param data: The data to be hashed.512 :type data: bytes513 :param hasher: The name of a hash implementation, supported by hashlib,514 or ``None``. Examples of valid values are ``'sha1'``,515 ``'sha224'``, ``'sha384'``, '``sha256'``, ``'md5'`` and516 ``'sha512'``. If no hasher is specified, the ``hasher``517 attribute of the :class:`InstalledDistribution` instance518 is used. If the hasher is determined to be ``None``, MD5519 is used as the hashing algorithm.520 :returns: The hash of the data. If a hasher was explicitly specified,521 the returned hash will be prefixed with the specified hasher522 followed by '='.523 :rtype: str524 """525 if hasher is None:526 hasher = self.hasher527 if hasher is None:528 hasher = hashlib.md5529 prefix = ''530 else:531 hasher = getattr(hashlib, hasher)532 prefix = '%s=' % self.hasher533 digest = hasher(data).digest()534 digest = base64.urlsafe_b64encode(digest).rstrip(b'=').decode('ascii')535 return '%s%s' % (prefix, digest)536 537 538class InstalledDistribution(BaseInstalledDistribution):539 """540 Created with the *path* of the ``.dist-info`` directory provided to the541 constructor. It reads the metadata contained in ``pydist.json`` when it is542 instantiated., or uses a passed in Metadata instance (useful for when543 dry-run mode is being used).544 """545 546 hasher = 'sha256'547 548 def __init__(self, path, metadata=None, env=None):549 self.modules = []550 self.finder = finder = resources.finder_for_path(path)551 if finder is None:552 raise ValueError('finder unavailable for %s' % path)553 if env and env._cache_enabled and path in env._cache.path:554 metadata = env._cache.path[path].metadata555 elif metadata is None:556 r = finder.find(METADATA_FILENAME)557 # Temporary - for Wheel 0.23 support558 if r is None:559 r = finder.find(WHEEL_METADATA_FILENAME)560 # Temporary - for legacy support561 if r is None:562 r = finder.find(LEGACY_METADATA_FILENAME)563 if r is None:564 raise ValueError('no %s found in %s' %565 (METADATA_FILENAME, path))566 with contextlib.closing(r.as_stream()) as stream:567 metadata = Metadata(fileobj=stream, scheme='legacy')568 569 super(InstalledDistribution, self).__init__(metadata, path, env)570 571 if env and env._cache_enabled:572 env._cache.add(self)573 574 r = finder.find('REQUESTED')575 self.requested = r is not None576 p = os.path.join(path, 'top_level.txt')577 if os.path.exists(p):578 with open(p, 'rb') as f:579 data = f.read().decode('utf-8')580 self.modules = data.splitlines()581 582 def __repr__(self):583 return '<InstalledDistribution %r %s at %r>' % (584 self.name, self.version, self.path)585 586 def __str__(self):587 return "%s %s" % (self.name, self.version)588 589 def _get_records(self):590 """591 Get the list of installed files for the distribution592 :return: A list of tuples of path, hash and size. Note that hash and593 size might be ``None`` for some entries. The path is exactly594 as stored in the file (which is as in PEP 376).595 """596 results = []597 r = self.get_distinfo_resource('RECORD')598 with contextlib.closing(r.as_stream()) as stream:599 with CSVReader(stream=stream) as record_reader:600 # Base location is parent dir of .dist-info dir601 # base_location = os.path.dirname(self.path)602 # base_location = os.path.abspath(base_location)603 for row in record_reader:604 missing = [None for i in range(len(row), 3)]605 path, checksum, size = row + missing606 # if not os.path.isabs(path):607 # path = path.replace('/', os.sep)608 # path = os.path.join(base_location, path)609 results.append((path, checksum, size))610 return results611 612 @cached_property613 def exports(self):614 """615 Return the information exported by this distribution.616 :return: A dictionary of exports, mapping an export category to a dict617 of :class:`ExportEntry` instances describing the individual618 export entries, and keyed by name.619 """620 result = {}621 r = self.get_distinfo_resource(EXPORTS_FILENAME)622 if r:623 result = self.read_exports()624 return result625 626 def read_exports(self):627 """628 Read exports data from a file in .ini format.629 630 :return: A dictionary of exports, mapping an export category to a list631 of :class:`ExportEntry` instances describing the individual632 export entries.633 """634 result = {}635 r = self.get_distinfo_resource(EXPORTS_FILENAME)636 if r:637 with contextlib.closing(r.as_stream()) as stream:638 result = read_exports(stream)639 return result640 641 def write_exports(self, exports):642 """643 Write a dictionary of exports to a file in .ini format.644 :param exports: A dictionary of exports, mapping an export category to645 a list of :class:`ExportEntry` instances describing the646 individual export entries.647 """648 rf = self.get_distinfo_file(EXPORTS_FILENAME)649 with open(rf, 'w') as f:650 write_exports(exports, f)651 652 def get_resource_path(self, relative_path):653 """654 NOTE: This API may change in the future.655 656 Return the absolute path to a resource file with the given relative657 path.658 659 :param relative_path: The path, relative to .dist-info, of the resource660 of interest.661 :return: The absolute path where the resource is to be found.662 """663 r = self.get_distinfo_resource('RESOURCES')664 with contextlib.closing(r.as_stream()) as stream:665 with CSVReader(stream=stream) as resources_reader:666 for relative, destination in resources_reader:667 if relative == relative_path:668 return destination669 raise KeyError('no resource file with relative path %r '670 'is installed' % relative_path)671 672 def list_installed_files(self):673 """674 Iterates over the ``RECORD`` entries and returns a tuple675 ``(path, hash, size)`` for each line.676 677 :returns: iterator of (path, hash, size)678 """679 for result in self._get_records():680 yield result681 682 def write_installed_files(self, paths, prefix, dry_run=False):683 """684 Writes the ``RECORD`` file, using the ``paths`` iterable passed in. Any685 existing ``RECORD`` file is silently overwritten.686 687 prefix is used to determine when to write absolute paths.688 """689 prefix = os.path.join(prefix, '')690 base = os.path.dirname(self.path)691 base_under_prefix = base.startswith(prefix)692 base = os.path.join(base, '')693 record_path = self.get_distinfo_file('RECORD')694 logger.info('creating %s', record_path)695 if dry_run:696 return None697 with CSVWriter(record_path) as writer:698 for path in paths:699 if os.path.isdir(path) or path.endswith(('.pyc', '.pyo')):700 # do not put size and hash, as in PEP-376701 hash_value = size = ''702 else:703 size = '%d' % os.path.getsize(path)704 with open(path, 'rb') as fp:705 hash_value = self.get_hash(fp.read())706 if path.startswith(base) or (base_under_prefix707 and path.startswith(prefix)):708 path = os.path.relpath(path, base)709 writer.writerow((path, hash_value, size))710 711 # add the RECORD file itself712 if record_path.startswith(base):713 record_path = os.path.relpath(record_path, base)714 writer.writerow((record_path, '', ''))715 return record_path716 717 def check_installed_files(self):718 """719 Checks that the hashes and sizes of the files in ``RECORD`` are720 matched by the files themselves. Returns a (possibly empty) list of721 mismatches. Each entry in the mismatch list will be a tuple consisting722 of the path, 'exists', 'size' or 'hash' according to what didn't match723 (existence is checked first, then size, then hash), the expected724 value and the actual value.725 """726 mismatches = []727 base = os.path.dirname(self.path)728 record_path = self.get_distinfo_file('RECORD')729 for path, hash_value, size in self.list_installed_files():730 if not os.path.isabs(path):731 path = os.path.join(base, path)732 if path == record_path:733 continue734 if not os.path.exists(path):735 mismatches.append((path, 'exists', True, False))736 elif os.path.isfile(path):737 actual_size = str(os.path.getsize(path))738 if size and actual_size != size:739 mismatches.append((path, 'size', size, actual_size))740 elif hash_value:741 if '=' in hash_value:742 hasher = hash_value.split('=', 1)[0]743 else:744 hasher = None745 746 with open(path, 'rb') as f:747 actual_hash = self.get_hash(f.read(), hasher)748 if actual_hash != hash_value:749 mismatches.append(750 (path, 'hash', hash_value, actual_hash))751 return mismatches752 753 @cached_property754 def shared_locations(self):755 """756 A dictionary of shared locations whose keys are in the set 'prefix',757 'purelib', 'platlib', 'scripts', 'headers', 'data' and 'namespace'.758 The corresponding value is the absolute path of that category for759 this distribution, and takes into account any paths selected by the760 user at installation time (e.g. via command-line arguments). In the761 case of the 'namespace' key, this would be a list of absolute paths762 for the roots of namespace packages in this distribution.763 764 The first time this property is accessed, the relevant information is765 read from the SHARED file in the .dist-info directory.766 """767 result = {}768 shared_path = os.path.join(self.path, 'SHARED')769 if os.path.isfile(shared_path):770 with codecs.open(shared_path, 'r', encoding='utf-8') as f:771 lines = f.read().splitlines()772 for line in lines:773 key, value = line.split('=', 1)774 if key == 'namespace':775 result.setdefault(key, []).append(value)776 else:777 result[key] = value778 return result779 780 def write_shared_locations(self, paths, dry_run=False):781 """782 Write shared location information to the SHARED file in .dist-info.783 :param paths: A dictionary as described in the documentation for784 :meth:`shared_locations`.785 :param dry_run: If True, the action is logged but no file is actually786 written.787 :return: The path of the file written to.788 """789 shared_path = os.path.join(self.path, 'SHARED')790 logger.info('creating %s', shared_path)791 if dry_run:792 return None793 lines = []794 for key in ('prefix', 'lib', 'headers', 'scripts', 'data'):795 path = paths[key]796 if os.path.isdir(paths[key]):797 lines.append('%s=%s' % (key, path))798 for ns in paths.get('namespace', ()):799 lines.append('namespace=%s' % ns)800 801 with codecs.open(shared_path, 'w', encoding='utf-8') as f:802 f.write('\n'.join(lines))803 return shared_path804 805 def get_distinfo_resource(self, path):806 if path not in DIST_FILES:807 raise DistlibException('invalid path for a dist-info file: '808 '%r at %r' % (path, self.path))809 finder = resources.finder_for_path(self.path)810 if finder is None:811 raise DistlibException('Unable to get a finder for %s' % self.path)812 return finder.find(path)813 814 def get_distinfo_file(self, path):815 """816 Returns a path located under the ``.dist-info`` directory. Returns a817 string representing the path.818 819 :parameter path: a ``'/'``-separated path relative to the820 ``.dist-info`` directory or an absolute path;821 If *path* is an absolute path and doesn't start822 with the ``.dist-info`` directory path,823 a :class:`DistlibException` is raised824 :type path: str825 :rtype: str826 """827 # Check if it is an absolute path # XXX use relpath, add tests828 if path.find(os.sep) >= 0:829 # it's an absolute path?830 distinfo_dirname, path = path.split(os.sep)[-2:]831 if distinfo_dirname != self.path.split(os.sep)[-1]:832 raise DistlibException(833 'dist-info file %r does not belong to the %r %s '834 'distribution' % (path, self.name, self.version))835 836 # The file must be relative837 if path not in DIST_FILES:838 raise DistlibException('invalid path for a dist-info file: '839 '%r at %r' % (path, self.path))840 841 return os.path.join(self.path, path)842 843 def list_distinfo_files(self):844 """845 Iterates over the ``RECORD`` entries and returns paths for each line if846 the path is pointing to a file located in the ``.dist-info`` directory847 or one of its subdirectories.848 849 :returns: iterator of paths850 """851 base = os.path.dirname(self.path)852 for path, checksum, size in self._get_records():853 # XXX add separator or use real relpath algo854 if not os.path.isabs(path):855 path = os.path.join(base, path)856 if path.startswith(self.path):857 yield path858 859 def __eq__(self, other):860 return (isinstance(other, InstalledDistribution)861 and self.path == other.path)862 863 # See http://docs.python.org/reference/datamodel#object.__hash__864 __hash__ = object.__hash__865 866 867class EggInfoDistribution(BaseInstalledDistribution):868 """Created with the *path* of the ``.egg-info`` directory or file provided869 to the constructor. It reads the metadata contained in the file itself, or870 if the given path happens to be a directory, the metadata is read from the871 file ``PKG-INFO`` under that directory."""872 873 requested = True # as we have no way of knowing, assume it was874 shared_locations = {}875 876 def __init__(self, path, env=None):877 878 def set_name_and_version(s, n, v):879 s.name = n880 s.key = n.lower() # for case-insensitive comparisons881 s.version = v882 883 self.path = path884 self.dist_path = env885 if env and env._cache_enabled and path in env._cache_egg.path:886 metadata = env._cache_egg.path[path].metadata887 set_name_and_version(self, metadata.name, metadata.version)888 else:889 metadata = self._get_metadata(path)890 891 # Need to be set before caching892 set_name_and_version(self, metadata.name, metadata.version)893 894 if env and env._cache_enabled:895 env._cache_egg.add(self)896 super(EggInfoDistribution, self).__init__(metadata, path, env)897 898 def _get_metadata(self, path):899 requires = None900 901 def parse_requires_data(data):902 """Create a list of dependencies from a requires.txt file.903 904 *data*: the contents of a setuptools-produced requires.txt file.905 """906 reqs = []907 lines = data.splitlines()908 for line in lines:909 line = line.strip()910 # sectioned files have bare newlines (separating sections)911 if not line: # pragma: no cover912 continue913 if line.startswith('['): # pragma: no cover914 logger.warning(915 'Unexpected line: quitting requirement scan: %r', line)916 break917 r = parse_requirement(line)918 if not r: # pragma: no cover919 logger.warning('Not recognised as a requirement: %r', line)920 continue921 if r.extras: # pragma: no cover922 logger.warning('extra requirements in requires.txt are '923 'not supported')924 if not r.constraints:925 reqs.append(r.name)926 else:927 cons = ', '.join('%s%s' % c for c in r.constraints)928 reqs.append('%s (%s)' % (r.name, cons))929 return reqs930 931 def parse_requires_path(req_path):932 """Create a list of dependencies from a requires.txt file.933 934 *req_path*: the path to a setuptools-produced requires.txt file.935 """936 937 reqs = []938 try:939 with codecs.open(req_path, 'r', 'utf-8') as fp:940 reqs = parse_requires_data(fp.read())941 except IOError:942 pass943 return reqs944 945 tl_path = tl_data = None946 if path.endswith('.egg'):947 if os.path.isdir(path):948 p = os.path.join(path, 'EGG-INFO')949 meta_path = os.path.join(p, 'PKG-INFO')950 metadata = Metadata(path=meta_path, scheme='legacy')951 req_path = os.path.join(p, 'requires.txt')952 tl_path = os.path.join(p, 'top_level.txt')953 requires = parse_requires_path(req_path)954 else:955 # FIXME handle the case where zipfile is not available956 zipf = zipimport.zipimporter(path)957 fileobj = StringIO(958 zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8'))959 metadata = Metadata(fileobj=fileobj, scheme='legacy')960 try:961 data = zipf.get_data('EGG-INFO/requires.txt')962 tl_data = zipf.get_data('EGG-INFO/top_level.txt').decode(963 'utf-8')964 requires = parse_requires_data(data.decode('utf-8'))965 except IOError:966 requires = None967 elif path.endswith('.egg-info'):968 if os.path.isdir(path):969 req_path = os.path.join(path, 'requires.txt')970 requires = parse_requires_path(req_path)971 path = os.path.join(path, 'PKG-INFO')972 tl_path = os.path.join(path, 'top_level.txt')973 metadata = Metadata(path=path, scheme='legacy')974 else:975 raise DistlibException('path must end with .egg-info or .egg, '976 'got %r' % path)977 978 if requires:979 metadata.add_requirements(requires)980 # look for top-level modules in top_level.txt, if present981 if tl_data is None:982 if tl_path is not None and os.path.exists(tl_path):983 with open(tl_path, 'rb') as f:984 tl_data = f.read().decode('utf-8')985 if not tl_data:986 tl_data = []987 else:988 tl_data = tl_data.splitlines()989 self.modules = tl_data990 return metadata991 992 def __repr__(self):993 return '<EggInfoDistribution %r %s at %r>' % (self.name, self.version,994 self.path)995 996 def __str__(self):997 return "%s %s" % (self.name, self.version)998 999 def check_installed_files(self):1000 """1001 Checks that the hashes and sizes of the files in ``RECORD`` are1002 matched by the files themselves. Returns a (possibly empty) list of1003 mismatches. Each entry in the mismatch list will be a tuple consisting1004 of the path, 'exists', 'size' or 'hash' according to what didn't match1005 (existence is checked first, then size, then hash), the expected1006 value and the actual value.1007 """1008 mismatches = []1009 record_path = os.path.join(self.path, 'installed-files.txt')1010 if os.path.exists(record_path):1011 for path, _, _ in self.list_installed_files():1012 if path == record_path:1013 continue1014 if not os.path.exists(path):1015 mismatches.append((path, 'exists', True, False))1016 return mismatches1017 1018 def list_installed_files(self):1019 """1020 Iterates over the ``installed-files.txt`` entries and returns a tuple1021 ``(path, hash, size)`` for each line.1022 1023 :returns: a list of (path, hash, size)1024 """1025 1026 def _md5(path):1027 f = open(path, 'rb')1028 try:1029 content = f.read()1030 finally:1031 f.close()1032 return hashlib.md5(content).hexdigest()1033 1034 def _size(path):1035 return os.stat(path).st_size1036 1037 record_path = os.path.join(self.path, 'installed-files.txt')1038 result = []1039 if os.path.exists(record_path):1040 with codecs.open(record_path, 'r', encoding='utf-8') as f:1041 for line in f:1042 line = line.strip()1043 p = os.path.normpath(os.path.join(self.path, line))1044 # "./" is present as a marker between installed files1045 # and installation metadata files1046 if not os.path.exists(p):1047 logger.warning('Non-existent file: %s', p)1048 if p.endswith(('.pyc', '.pyo')):1049 continue1050 # otherwise fall through and fail1051 if not os.path.isdir(p):1052 result.append((p, _md5(p), _size(p)))1053 result.append((record_path, None, None))1054 return result1055 1056 def list_distinfo_files(self, absolute=False):1057 """1058 Iterates over the ``installed-files.txt`` entries and returns paths for1059 each line if the path is pointing to a file located in the1060 ``.egg-info`` directory or one of its subdirectories.1061 1062 :parameter absolute: If *absolute* is ``True``, each returned path is1063 transformed into a local absolute path. Otherwise the1064 raw value from ``installed-files.txt`` is returned.1065 :type absolute: boolean1066 :returns: iterator of paths1067 """1068 record_path = os.path.join(self.path, 'installed-files.txt')1069 if os.path.exists(record_path):1070 skip = True1071 with codecs.open(record_path, 'r', encoding='utf-8') as f:1072 for line in f:1073 line = line.strip()1074 if line == './':1075 skip = False1076 continue1077 if not skip:1078 p = os.path.normpath(os.path.join(self.path, line))1079 if p.startswith(self.path):1080 if absolute:1081 yield p1082 else:1083 yield line1084 1085 def __eq__(self, other):1086 return (isinstance(other, EggInfoDistribution)1087 and self.path == other.path)1088 1089 # See http://docs.python.org/reference/datamodel#object.__hash__1090 __hash__ = object.__hash__1091 1092 1093new_dist_class = InstalledDistribution1094old_dist_class = EggInfoDistribution1095 1096 1097class DependencyGraph(object):1098 """1099 Represents a dependency graph between distributions.1100 1101 The dependency relationships are stored in an ``adjacency_list`` that maps1102 distributions to a list of ``(other, label)`` tuples where ``other``1103 is a distribution and the edge is labeled with ``label`` (i.e. the version1104 specifier, if such was provided). Also, for more efficient traversal, for1105 every distribution ``x``, a list of predecessors is kept in1106 ``reverse_list[x]``. An edge from distribution ``a`` to1107 distribution ``b`` means that ``a`` depends on ``b``. If any missing1108 dependencies are found, they are stored in ``missing``, which is a1109 dictionary that maps distributions to a list of requirements that were not1110 provided by any other distributions.1111 """1112 1113 def __init__(self):1114 self.adjacency_list = {}1115 self.reverse_list = {}1116 self.missing = {}1117 1118 def add_distribution(self, distribution):1119 """Add the *distribution* to the graph.1120 1121 :type distribution: :class:`distutils2.database.InstalledDistribution`1122 or :class:`distutils2.database.EggInfoDistribution`1123 """1124 self.adjacency_list[distribution] = []1125 self.reverse_list[distribution] = []1126 # self.missing[distribution] = []1127 1128 def add_edge(self, x, y, label=None):1129 """Add an edge from distribution *x* to distribution *y* with the given1130 *label*.1131 1132 :type x: :class:`distutils2.database.InstalledDistribution` or1133 :class:`distutils2.database.EggInfoDistribution`1134 :type y: :class:`distutils2.database.InstalledDistribution` or1135 :class:`distutils2.database.EggInfoDistribution`1136 :type label: ``str`` or ``None``1137 """1138 self.adjacency_list[x].append((y, label))1139 # multiple edges are allowed, so be careful1140 if x not in self.reverse_list[y]:1141 self.reverse_list[y].append(x)1142 1143 def add_missing(self, distribution, requirement):1144 """1145 Add a missing *requirement* for the given *distribution*.1146 1147 :type distribution: :class:`distutils2.database.InstalledDistribution`1148 or :class:`distutils2.database.EggInfoDistribution`1149 :type requirement: ``str``1150 """1151 logger.debug('%s missing %r', distribution, requirement)1152 self.missing.setdefault(distribution, []).append(requirement)1153 1154 def _repr_dist(self, dist):1155 return '%s %s' % (dist.name, dist.version)1156 1157 def repr_node(self, dist, level=1):1158 """Prints only a subgraph"""1159 output = [self._repr_dist(dist)]1160 for other, label in self.adjacency_list[dist]:1161 dist = self._repr_dist(other)1162 if label is not None:1163 dist = '%s [%s]' % (dist, label)1164 output.append(' ' * level + str(dist))1165 suboutput = self.repr_node(other, level + 1)1166 subs = suboutput.split('\n')1167 output.extend(subs[1:])1168 return '\n'.join(output)1169 1170 def to_dot(self, f, skip_disconnected=True):1171 """Writes a DOT output for the graph to the provided file *f*.1172 1173 If *skip_disconnected* is set to ``True``, then all distributions1174 that are not dependent on any other distribution are skipped.1175 1176 :type f: has to support ``file``-like operations1177 :type skip_disconnected: ``bool``1178 """1179 disconnected = []1180 1181 f.write("digraph dependencies {\n")1182 for dist, adjs in self.adjacency_list.items():1183 if len(adjs) == 0 and not skip_disconnected:1184 disconnected.append(dist)1185 for other, label in adjs:1186 if label is not None:1187 f.write('"%s" -> "%s" [label="%s"]\n' %1188 (dist.name, other.name, label))1189 else:1190 f.write('"%s" -> "%s"\n' % (dist.name, other.name))1191 if not skip_disconnected and len(disconnected) > 0:1192 f.write('subgraph disconnected {\n')1193 f.write('label = "Disconnected"\n')1194 f.write('bgcolor = red\n')1195 1196 for dist in disconnected:1197 f.write('"%s"' % dist.name)1198 f.write('\n')1199 f.write('}\n')1200 f.write('}\n')