codekingpro/portable-devtools

1"""2Load setuptools configuration from ``pyproject.toml`` files.3 4**PRIVATE MODULE**: API reserved for setuptools internal usage only.5 6To read project metadata, consider using7``build.util.project_wheel_metadata`` (https://pypi.org/project/build/).8For simple scenarios, you can also try parsing the file directly9with the help of ``tomllib`` or ``tomli``.10"""11 12from __future__ import annotations13 14import logging15import os16from collections.abc import Mapping17from contextlib import contextmanager18from functools import partial19from types import TracebackType20from typing import TYPE_CHECKING, Any, Callable21 22from .._path import StrPath23from ..errors import FileError, InvalidConfigError24from ..warnings import SetuptoolsWarning25from . import expand as _expand26from ._apply_pyprojecttoml import _PREVIOUSLY_DEFINED, _MissingDynamic, apply as _apply27 28if TYPE_CHECKING:29    from typing_extensions import Self30 31    from setuptools.dist import Distribution32 33_logger = logging.getLogger(__name__)34 35 36def load_file(filepath: StrPath) -> dict:37    from ..compat.py310 import tomllib38 39    with open(filepath, "rb") as file:40        return tomllib.load(file)41 42 43def validate(config: dict, filepath: StrPath) -> bool:44    from . import _validate_pyproject as validator45 46    trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier")47    if hasattr(trove_classifier, "_disable_download"):48        # Improve reproducibility by default. See abravalheri/validate-pyproject#3149        trove_classifier._disable_download()  # type: ignore[union-attr]50 51    try:52        return validator.validate(config)53    except validator.ValidationError as ex:54        summary = f"configuration error: {ex.summary}"55        if ex.name.strip("`") != "project":56            # Probably it is just a field missing/misnamed, not worthy the verbosity...57            _logger.debug(summary)58            _logger.debug(ex.details)59 60        error = f"invalid pyproject.toml config: {ex.name}."61        raise ValueError(f"{error}\n{summary}") from None62 63 64def apply_configuration(65    dist: Distribution,66    filepath: StrPath,67    ignore_option_errors: bool = False,68) -> Distribution:69    """Apply the configuration from a ``pyproject.toml`` file into an existing70    distribution object.71    """72    config = read_configuration(filepath, True, ignore_option_errors, dist)73    return _apply(dist, config, filepath)74 75 76def read_configuration(77    filepath: StrPath,78    expand: bool = True,79    ignore_option_errors: bool = False,80    dist: Distribution | None = None,81) -> dict[str, Any]:82    """Read given configuration file and returns options from it as a dict.83 84    :param str|unicode filepath: Path to configuration file in the ``pyproject.toml``85        format.86 87    :param bool expand: Whether to expand directives and other computed values88        (i.e. post-process the given configuration)89 90    :param bool ignore_option_errors: Whether to silently ignore91        options, values of which could not be resolved (e.g. due to exceptions92        in directives such as file:, attr:, etc.).93        If False exceptions are propagated as expected.94 95    :param Distribution|None: Distribution object to which the configuration refers.96        If not given a dummy object will be created and discarded after the97        configuration is read. This is used for auto-discovery of packages and in the98        case a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.99        When ``expand=False`` this object is simply ignored.100 101    :rtype: dict102    """103    filepath = os.path.abspath(filepath)104 105    if not os.path.isfile(filepath):106        raise FileError(f"Configuration file {filepath!r} does not exist.")107 108    asdict = load_file(filepath) or {}109    project_table = asdict.get("project", {})110    tool_table = asdict.get("tool", {})111    setuptools_table = tool_table.get("setuptools", {})112    if not asdict or not (project_table or setuptools_table):113        return {}  # User is not using pyproject to configure setuptools114 115    if "setuptools" in asdict.get("tools", {}):116        # let the user know they probably have a typo in their metadata117        _ToolsTypoInMetadata.emit()118 119    if "distutils" in tool_table:120        _ExperimentalConfiguration.emit(subject="[tool.distutils]")121 122    # There is an overall sense in the community that making include_package_data=True123    # the default would be an improvement.124    # `ini2toml` backfills include_package_data=False when nothing is explicitly given,125    # therefore setting a default here is backwards compatible.126    if dist and dist.include_package_data is not None:127        setuptools_table.setdefault("include-package-data", dist.include_package_data)128    else:129        setuptools_table.setdefault("include-package-data", True)130    # Persist changes:131    asdict["tool"] = tool_table132    tool_table["setuptools"] = setuptools_table133 134    if "ext-modules" in setuptools_table:135        _ExperimentalConfiguration.emit(subject="[tool.setuptools.ext-modules]")136 137    fields = ("import-names", "import-namespaces")138    places = (project_table, project_table.get("dynamic", []))139    if any(field in place for field in fields for place in places):140        raise NotImplementedError(141            "Setuptools does not support `import-names` and `import-namespaces`"142            " in `pyproject.toml` yet. If your are interested in this feature, "143            " please consider submitting a contribution via pull requests."144        )145 146    with _ignore_errors(ignore_option_errors):147        # Don't complain about unrelated errors (e.g. tools not using the "tool" table)148        subset = {"project": project_table, "tool": {"setuptools": setuptools_table}}149        validate(subset, filepath)150 151    if expand:152        root_dir = os.path.dirname(filepath)153        return expand_configuration(asdict, root_dir, ignore_option_errors, dist)154 155    return asdict156 157 158def expand_configuration(159    config: dict,160    root_dir: StrPath | None = None,161    ignore_option_errors: bool = False,162    dist: Distribution | None = None,163) -> dict:164    """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...)165    find their final values.166 167    :param dict config: Dict containing the configuration for the distribution168    :param str root_dir: Top-level directory for the distribution/project169        (the same directory where ``pyproject.toml`` is place)170    :param bool ignore_option_errors: see :func:`read_configuration`171    :param Distribution|None: Distribution object to which the configuration refers.172        If not given a dummy object will be created and discarded after the173        configuration is read. Used in the case a dynamic configuration174        (e.g. ``attr`` or ``cmdclass``).175 176    :rtype: dict177    """178    return _ConfigExpander(config, root_dir, ignore_option_errors, dist).expand()179 180 181class _ConfigExpander:182    def __init__(183        self,184        config: dict,185        root_dir: StrPath | None = None,186        ignore_option_errors: bool = False,187        dist: Distribution | None = None,188    ) -> None:189        self.config = config190        self.root_dir = root_dir or os.getcwd()191        self.project_cfg = config.get("project", {})192        self.dynamic = self.project_cfg.get("dynamic", [])193        self.setuptools_cfg = config.get("tool", {}).get("setuptools", {})194        self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})195        self.ignore_option_errors = ignore_option_errors196        self._dist = dist197        self._referenced_files = set[str]()198 199    def _ensure_dist(self) -> Distribution:200        from setuptools.dist import Distribution201 202        attrs = {"src_root": self.root_dir, "name": self.project_cfg.get("name", None)}203        return self._dist or Distribution(attrs)204 205    def _process_field(self, container: dict, field: str, fn: Callable):206        if field in container:207            with _ignore_errors(self.ignore_option_errors):208                container[field] = fn(container[field])209 210    def _canonic_package_data(self, field="package-data"):211        package_data = self.setuptools_cfg.get(field, {})212        return _expand.canonic_package_data(package_data)213 214    def expand(self):215        self._expand_packages()216        self._canonic_package_data()217        self._canonic_package_data("exclude-package-data")218 219        # A distribution object is required for discovering the correct package_dir220        dist = self._ensure_dist()221        ctx = _EnsurePackagesDiscovered(dist, self.project_cfg, self.setuptools_cfg)222        with ctx as ensure_discovered:223            package_dir = ensure_discovered.package_dir224            self._expand_data_files()225            self._expand_cmdclass(package_dir)226            self._expand_all_dynamic(dist, package_dir)227 228        dist._referenced_files.update(self._referenced_files)229        return self.config230 231    def _expand_packages(self):232        packages = self.setuptools_cfg.get("packages")233        if packages is None or isinstance(packages, (list, tuple)):234            return235 236        find = packages.get("find")237        if isinstance(find, dict):238            find["root_dir"] = self.root_dir239            find["fill_package_dir"] = self.setuptools_cfg.setdefault("package-dir", {})240            with _ignore_errors(self.ignore_option_errors):241                self.setuptools_cfg["packages"] = _expand.find_packages(**find)242 243    def _expand_data_files(self):244        data_files = partial(_expand.canonic_data_files, root_dir=self.root_dir)245        self._process_field(self.setuptools_cfg, "data-files", data_files)246 247    def _expand_cmdclass(self, package_dir: Mapping[str, str]):248        root_dir = self.root_dir249        cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir)250        self._process_field(self.setuptools_cfg, "cmdclass", cmdclass)251 252    def _expand_all_dynamic(self, dist: Distribution, package_dir: Mapping[str, str]):253        special = (  # need special handling254            "version",255            "readme",256            "entry-points",257            "scripts",258            "gui-scripts",259            "classifiers",260            "dependencies",261            "optional-dependencies",262        )263        # `_obtain` functions are assumed to raise appropriate exceptions/warnings.264        obtained_dynamic = {265            field: self._obtain(dist, field, package_dir)266            for field in self.dynamic267            if field not in special268        }269        obtained_dynamic.update(270            self._obtain_entry_points(dist, package_dir) or {},271            version=self._obtain_version(dist, package_dir),272            readme=self._obtain_readme(dist),273            classifiers=self._obtain_classifiers(dist),274            dependencies=self._obtain_dependencies(dist),275            optional_dependencies=self._obtain_optional_dependencies(dist),276        )277        # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value278        # might have already been set by setup.py/extensions, so avoid overwriting.279        updates = {k: v for k, v in obtained_dynamic.items() if v is not None}280        self.project_cfg.update(updates)281 282    def _ensure_previously_set(self, dist: Distribution, field: str):283        previous = _PREVIOUSLY_DEFINED[field](dist)284        if previous is None and not self.ignore_option_errors:285            msg = (286                f"No configuration found for dynamic {field!r}.\n"287                "Some dynamic fields need to be specified via `tool.setuptools.dynamic`"288                "\nothers must be specified via the equivalent attribute in `setup.py`."289            )290            raise InvalidConfigError(msg)291 292    def _expand_directive(293        self, specifier: str, directive, package_dir: Mapping[str, str]294    ):295        from more_itertools import always_iterable296 297        with _ignore_errors(self.ignore_option_errors):298            root_dir = self.root_dir299            if "file" in directive:300                self._referenced_files.update(always_iterable(directive["file"]))301                return _expand.read_files(directive["file"], root_dir)302            if "attr" in directive:303                return _expand.read_attr(directive["attr"], package_dir, root_dir)304            raise ValueError(f"invalid `{specifier}`: {directive!r}")305        return None306 307    def _obtain(self, dist: Distribution, field: str, package_dir: Mapping[str, str]):308        if field in self.dynamic_cfg:309            return self._expand_directive(310                f"tool.setuptools.dynamic.{field}",311                self.dynamic_cfg[field],312                package_dir,313            )314        self._ensure_previously_set(dist, field)315        return None316 317    def _obtain_version(self, dist: Distribution, package_dir: Mapping[str, str]):318        # Since plugins can set version, let's silently skip if it cannot be obtained319        if "version" in self.dynamic and "version" in self.dynamic_cfg:320            return _expand.version(321                # We already do an early check for the presence of "version"322                self._obtain(dist, "version", package_dir)  # pyright: ignore[reportArgumentType]323            )324        return None325 326    def _obtain_readme(self, dist: Distribution) -> dict[str, str] | None:327        if "readme" not in self.dynamic:328            return None329 330        dynamic_cfg = self.dynamic_cfg331        if "readme" in dynamic_cfg:332            return {333                # We already do an early check for the presence of "readme"334                "text": self._obtain(dist, "readme", {}),335                "content-type": dynamic_cfg["readme"].get("content-type", "text/x-rst"),336            }  # pyright: ignore[reportReturnType]337 338        self._ensure_previously_set(dist, "readme")339        return None340 341    def _obtain_entry_points(342        self, dist: Distribution, package_dir: Mapping[str, str]343    ) -> dict[str, dict[str, Any]] | None:344        fields = ("entry-points", "scripts", "gui-scripts")345        if not any(field in self.dynamic for field in fields):346            return None347 348        text = self._obtain(dist, "entry-points", package_dir)349        if text is None:350            return None351 352        groups = _expand.entry_points(text)353        # Any is str | dict[str, str], but causes variance issues354        expanded: dict[str, dict[str, Any]] = {"entry-points": groups}355 356        def _set_scripts(field: str, group: str):357            if group in groups:358                value = groups.pop(group)359                if field not in self.dynamic:360                    raise InvalidConfigError(_MissingDynamic.details(field, value))361                expanded[field] = value362 363        _set_scripts("scripts", "console_scripts")364        _set_scripts("gui-scripts", "gui_scripts")365 366        return expanded367 368    def _obtain_classifiers(self, dist: Distribution):369        if "classifiers" in self.dynamic:370            value = self._obtain(dist, "classifiers", {})371            if value:372                return value.splitlines()373        return None374 375    def _obtain_dependencies(self, dist: Distribution):376        if "dependencies" in self.dynamic:377            value = self._obtain(dist, "dependencies", {})378            if value:379                return _parse_requirements_list(value)380        return None381 382    def _obtain_optional_dependencies(self, dist: Distribution):383        if "optional-dependencies" not in self.dynamic:384            return None385        if "optional-dependencies" in self.dynamic_cfg:386            optional_dependencies_map = self.dynamic_cfg["optional-dependencies"]387            assert isinstance(optional_dependencies_map, dict)388            return {389                group: _parse_requirements_list(390                    self._expand_directive(391                        f"tool.setuptools.dynamic.optional-dependencies.{group}",392                        directive,393                        {},394                    )395                )396                for group, directive in optional_dependencies_map.items()397            }398        self._ensure_previously_set(dist, "optional-dependencies")399        return None400 401 402def _parse_requirements_list(value):403    return [404        line405        for line in value.splitlines()406        if line.strip() and not line.strip().startswith("#")407    ]408 409 410@contextmanager411def _ignore_errors(ignore_option_errors: bool):412    if not ignore_option_errors:413        yield414        return415 416    try:417        yield418    except Exception as ex:419        _logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}")420 421 422class _EnsurePackagesDiscovered(_expand.EnsurePackagesDiscovered):423    def __init__(424        self, distribution: Distribution, project_cfg: dict, setuptools_cfg: dict425    ) -> None:426        super().__init__(distribution)427        self._project_cfg = project_cfg428        self._setuptools_cfg = setuptools_cfg429 430    def __enter__(self) -> Self:431        """When entering the context, the values of ``packages``, ``py_modules`` and432        ``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``.433        """434        dist, cfg = self._dist, self._setuptools_cfg435        package_dir: dict[str, str] = cfg.setdefault("package-dir", {})436        package_dir.update(dist.package_dir or {})437        dist.package_dir = package_dir  # needs to be the same object438 439        dist.set_defaults._ignore_ext_modules()  # pyproject.toml-specific behaviour440 441        # Set `name`, `py_modules` and `packages` in dist to short-circuit442        # auto-discovery, but avoid overwriting empty lists purposefully set by users.443        if dist.metadata.name is None:444            dist.metadata.name = self._project_cfg.get("name")445        if dist.py_modules is None:446            dist.py_modules = cfg.get("py-modules")447        if dist.packages is None:448            dist.packages = cfg.get("packages")449 450        return super().__enter__()451 452    def __exit__(453        self,454        exc_type: type[BaseException] | None,455        exc_value: BaseException | None,456        traceback: TracebackType | None,457    ) -> None:458        """When exiting the context, if values of ``packages``, ``py_modules`` and459        ``package_dir`` are missing in ``setuptools_cfg``, copy from ``dist``.460        """461        # If anything was discovered set them back, so they count in the final config.462        self._setuptools_cfg.setdefault("packages", self._dist.packages)463        self._setuptools_cfg.setdefault("py-modules", self._dist.py_modules)464        return super().__exit__(exc_type, exc_value, traceback)465 466 467class _ExperimentalConfiguration(SetuptoolsWarning):468    _SUMMARY = (469        "`{subject}` in `pyproject.toml` is still *experimental* "470        "and likely to change in future releases."471    )472 473 474class _ToolsTypoInMetadata(SetuptoolsWarning):475    _SUMMARY = (476        "Ignoring [tools.setuptools] in pyproject.toml, did you mean [tool.setuptools]?"477    )478