codekingpro/portable-devtools

1"""2Load setuptools configuration from ``pyproject.toml`` files.3 4**PRIVATE MODULE**: API reserved for setuptools internal usage only.5 6To read project metadata, consider using7``build.util.project_wheel_metadata`` (https://pypi.org/project/build/).8For simple scenarios, you can also try parsing the file directly9with the help of ``tomllib`` or ``tomli``.10"""11 12import logging13import os14from contextlib import contextmanager15from functools import partial16from typing import TYPE_CHECKING, Callable, Dict, Mapping, Optional, Set17 18from .._path import StrPath19from ..errors import FileError, InvalidConfigError20from ..warnings import SetuptoolsWarning21from . import expand as _expand22from ._apply_pyprojecttoml import _PREVIOUSLY_DEFINED, _MissingDynamic23from ._apply_pyprojecttoml import apply as _apply24 25if TYPE_CHECKING:26    from setuptools.dist import Distribution  # noqa27    from typing_extensions import Self28 29_logger = logging.getLogger(__name__)30 31 32def load_file(filepath: StrPath) -> dict:33    from ..compat.py310 import tomllib34 35    with open(filepath, "rb") as file:36        return tomllib.load(file)37 38 39def validate(config: dict, filepath: StrPath) -> bool:40    from . import _validate_pyproject as validator41 42    trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier")43    if hasattr(trove_classifier, "_disable_download"):44        # Improve reproducibility by default. See issue 31 for validate-pyproject.45        trove_classifier._disable_download()  # type: ignore46 47    try:48        return validator.validate(config)49    except validator.ValidationError as ex:50        summary = f"configuration error: {ex.summary}"51        if ex.name.strip("`") != "project":52            # Probably it is just a field missing/misnamed, not worthy the verbosity...53            _logger.debug(summary)54            _logger.debug(ex.details)55 56        error = f"invalid pyproject.toml config: {ex.name}."57        raise ValueError(f"{error}\n{summary}") from None58 59 60def apply_configuration(61    dist: "Distribution",62    filepath: StrPath,63    ignore_option_errors=False,64) -> "Distribution":65    """Apply the configuration from a ``pyproject.toml`` file into an existing66    distribution object.67    """68    config = read_configuration(filepath, True, ignore_option_errors, dist)69    return _apply(dist, config, filepath)70 71 72def read_configuration(73    filepath: StrPath,74    expand=True,75    ignore_option_errors=False,76    dist: Optional["Distribution"] = None,77):78    """Read given configuration file and returns options from it as a dict.79 80    :param str|unicode filepath: Path to configuration file in the ``pyproject.toml``81        format.82 83    :param bool expand: Whether to expand directives and other computed values84        (i.e. post-process the given configuration)85 86    :param bool ignore_option_errors: Whether to silently ignore87        options, values of which could not be resolved (e.g. due to exceptions88        in directives such as file:, attr:, etc.).89        If False exceptions are propagated as expected.90 91    :param Distribution|None: Distribution object to which the configuration refers.92        If not given a dummy object will be created and discarded after the93        configuration is read. This is used for auto-discovery of packages and in the94        case a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.95        When ``expand=False`` this object is simply ignored.96 97    :rtype: dict98    """99    filepath = os.path.abspath(filepath)100 101    if not os.path.isfile(filepath):102        raise FileError(f"Configuration file {filepath!r} does not exist.")103 104    asdict = load_file(filepath) or {}105    project_table = asdict.get("project", {})106    tool_table = asdict.get("tool", {})107    setuptools_table = tool_table.get("setuptools", {})108    if not asdict or not (project_table or setuptools_table):109        return {}  # User is not using pyproject to configure setuptools110 111    if "distutils" in tool_table:112        _ExperimentalConfiguration.emit(subject="[tool.distutils]")113 114    # There is an overall sense in the community that making include_package_data=True115    # the default would be an improvement.116    # `ini2toml` backfills include_package_data=False when nothing is explicitly given,117    # therefore setting a default here is backwards compatible.118    if dist and getattr(dist, "include_package_data", None) is not None:119        setuptools_table.setdefault("include-package-data", dist.include_package_data)120    else:121        setuptools_table.setdefault("include-package-data", True)122    # Persist changes:123    asdict["tool"] = tool_table124    tool_table["setuptools"] = setuptools_table125 126    with _ignore_errors(ignore_option_errors):127        # Don't complain about unrelated errors (e.g. tools not using the "tool" table)128        subset = {"project": project_table, "tool": {"setuptools": setuptools_table}}129        validate(subset, filepath)130 131    if expand:132        root_dir = os.path.dirname(filepath)133        return expand_configuration(asdict, root_dir, ignore_option_errors, dist)134 135    return asdict136 137 138def expand_configuration(139    config: dict,140    root_dir: Optional[StrPath] = None,141    ignore_option_errors: bool = False,142    dist: Optional["Distribution"] = None,143) -> dict:144    """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...)145    find their final values.146 147    :param dict config: Dict containing the configuration for the distribution148    :param str root_dir: Top-level directory for the distribution/project149        (the same directory where ``pyproject.toml`` is place)150    :param bool ignore_option_errors: see :func:`read_configuration`151    :param Distribution|None: Distribution object to which the configuration refers.152        If not given a dummy object will be created and discarded after the153        configuration is read. Used in the case a dynamic configuration154        (e.g. ``attr`` or ``cmdclass``).155 156    :rtype: dict157    """158    return _ConfigExpander(config, root_dir, ignore_option_errors, dist).expand()159 160 161class _ConfigExpander:162    def __init__(163        self,164        config: dict,165        root_dir: Optional[StrPath] = None,166        ignore_option_errors: bool = False,167        dist: Optional["Distribution"] = None,168    ):169        self.config = config170        self.root_dir = root_dir or os.getcwd()171        self.project_cfg = config.get("project", {})172        self.dynamic = self.project_cfg.get("dynamic", [])173        self.setuptools_cfg = config.get("tool", {}).get("setuptools", {})174        self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})175        self.ignore_option_errors = ignore_option_errors176        self._dist = dist177        self._referenced_files: Set[str] = set()178 179    def _ensure_dist(self) -> "Distribution":180        from setuptools.dist import Distribution181 182        attrs = {"src_root": self.root_dir, "name": self.project_cfg.get("name", None)}183        return self._dist or Distribution(attrs)184 185    def _process_field(self, container: dict, field: str, fn: Callable):186        if field in container:187            with _ignore_errors(self.ignore_option_errors):188                container[field] = fn(container[field])189 190    def _canonic_package_data(self, field="package-data"):191        package_data = self.setuptools_cfg.get(field, {})192        return _expand.canonic_package_data(package_data)193 194    def expand(self):195        self._expand_packages()196        self._canonic_package_data()197        self._canonic_package_data("exclude-package-data")198 199        # A distribution object is required for discovering the correct package_dir200        dist = self._ensure_dist()201        ctx = _EnsurePackagesDiscovered(dist, self.project_cfg, self.setuptools_cfg)202        with ctx as ensure_discovered:203            package_dir = ensure_discovered.package_dir204            self._expand_data_files()205            self._expand_cmdclass(package_dir)206            self._expand_all_dynamic(dist, package_dir)207 208        dist._referenced_files.update(self._referenced_files)209        return self.config210 211    def _expand_packages(self):212        packages = self.setuptools_cfg.get("packages")213        if packages is None or isinstance(packages, (list, tuple)):214            return215 216        find = packages.get("find")217        if isinstance(find, dict):218            find["root_dir"] = self.root_dir219            find["fill_package_dir"] = self.setuptools_cfg.setdefault("package-dir", {})220            with _ignore_errors(self.ignore_option_errors):221                self.setuptools_cfg["packages"] = _expand.find_packages(**find)222 223    def _expand_data_files(self):224        data_files = partial(_expand.canonic_data_files, root_dir=self.root_dir)225        self._process_field(self.setuptools_cfg, "data-files", data_files)226 227    def _expand_cmdclass(self, package_dir: Mapping[str, str]):228        root_dir = self.root_dir229        cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir)230        self._process_field(self.setuptools_cfg, "cmdclass", cmdclass)231 232    def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, str]):233        special = (  # need special handling234            "version",235            "readme",236            "entry-points",237            "scripts",238            "gui-scripts",239            "classifiers",240            "dependencies",241            "optional-dependencies",242        )243        # `_obtain` functions are assumed to raise appropriate exceptions/warnings.244        obtained_dynamic = {245            field: self._obtain(dist, field, package_dir)246            for field in self.dynamic247            if field not in special248        }249        obtained_dynamic.update(250            self._obtain_entry_points(dist, package_dir) or {},251            version=self._obtain_version(dist, package_dir),252            readme=self._obtain_readme(dist),253            classifiers=self._obtain_classifiers(dist),254            dependencies=self._obtain_dependencies(dist),255            optional_dependencies=self._obtain_optional_dependencies(dist),256        )257        # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value258        # might have already been set by setup.py/extensions, so avoid overwriting.259        updates = {k: v for k, v in obtained_dynamic.items() if v is not None}260        self.project_cfg.update(updates)261 262    def _ensure_previously_set(self, dist: "Distribution", field: str):263        previous = _PREVIOUSLY_DEFINED[field](dist)264        if previous is None and not self.ignore_option_errors:265            msg = (266                f"No configuration found for dynamic {field!r}.\n"267                "Some dynamic fields need to be specified via `tool.setuptools.dynamic`"268                "\nothers must be specified via the equivalent attribute in `setup.py`."269            )270            raise InvalidConfigError(msg)271 272    def _expand_directive(273        self, specifier: str, directive, package_dir: Mapping[str, str]274    ):275        from setuptools.extern.more_itertools import always_iterable276 277        with _ignore_errors(self.ignore_option_errors):278            root_dir = self.root_dir279            if "file" in directive:280                self._referenced_files.update(always_iterable(directive["file"]))281                return _expand.read_files(directive["file"], root_dir)282            if "attr" in directive:283                return _expand.read_attr(directive["attr"], package_dir, root_dir)284            raise ValueError(f"invalid `{specifier}`: {directive!r}")285        return None286 287    def _obtain(self, dist: "Distribution", field: str, package_dir: Mapping[str, str]):288        if field in self.dynamic_cfg:289            return self._expand_directive(290                f"tool.setuptools.dynamic.{field}",291                self.dynamic_cfg[field],292                package_dir,293            )294        self._ensure_previously_set(dist, field)295        return None296 297    def _obtain_version(self, dist: "Distribution", package_dir: Mapping[str, str]):298        # Since plugins can set version, let's silently skip if it cannot be obtained299        if "version" in self.dynamic and "version" in self.dynamic_cfg:300            return _expand.version(self._obtain(dist, "version", package_dir))301        return None302 303    def _obtain_readme(self, dist: "Distribution") -> Optional[Dict[str, str]]:304        if "readme" not in self.dynamic:305            return None306 307        dynamic_cfg = self.dynamic_cfg308        if "readme" in dynamic_cfg:309            return {310                "text": self._obtain(dist, "readme", {}),311                "content-type": dynamic_cfg["readme"].get("content-type", "text/x-rst"),312            }313 314        self._ensure_previously_set(dist, "readme")315        return None316 317    def _obtain_entry_points(318        self, dist: "Distribution", package_dir: Mapping[str, str]319    ) -> Optional[Dict[str, dict]]:320        fields = ("entry-points", "scripts", "gui-scripts")321        if not any(field in self.dynamic for field in fields):322            return None323 324        text = self._obtain(dist, "entry-points", package_dir)325        if text is None:326            return None327 328        groups = _expand.entry_points(text)329        expanded = {"entry-points": groups}330 331        def _set_scripts(field: str, group: str):332            if group in groups:333                value = groups.pop(group)334                if field not in self.dynamic:335                    raise InvalidConfigError(_MissingDynamic.details(field, value))336                expanded[field] = value337 338        _set_scripts("scripts", "console_scripts")339        _set_scripts("gui-scripts", "gui_scripts")340 341        return expanded342 343    def _obtain_classifiers(self, dist: "Distribution"):344        if "classifiers" in self.dynamic:345            value = self._obtain(dist, "classifiers", {})346            if value:347                return value.splitlines()348        return None349 350    def _obtain_dependencies(self, dist: "Distribution"):351        if "dependencies" in self.dynamic:352            value = self._obtain(dist, "dependencies", {})353            if value:354                return _parse_requirements_list(value)355        return None356 357    def _obtain_optional_dependencies(self, dist: "Distribution"):358        if "optional-dependencies" not in self.dynamic:359            return None360        if "optional-dependencies" in self.dynamic_cfg:361            optional_dependencies_map = self.dynamic_cfg["optional-dependencies"]362            assert isinstance(optional_dependencies_map, dict)363            return {364                group: _parse_requirements_list(365                    self._expand_directive(366                        f"tool.setuptools.dynamic.optional-dependencies.{group}",367                        directive,368                        {},369                    )370                )371                for group, directive in optional_dependencies_map.items()372            }373        self._ensure_previously_set(dist, "optional-dependencies")374        return None375 376 377def _parse_requirements_list(value):378    return [379        line380        for line in value.splitlines()381        if line.strip() and not line.strip().startswith("#")382    ]383 384 385@contextmanager386def _ignore_errors(ignore_option_errors: bool):387    if not ignore_option_errors:388        yield389        return390 391    try:392        yield393    except Exception as ex:394        _logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}")395 396 397class _EnsurePackagesDiscovered(_expand.EnsurePackagesDiscovered):398    def __init__(399        self, distribution: "Distribution", project_cfg: dict, setuptools_cfg: dict400    ):401        super().__init__(distribution)402        self._project_cfg = project_cfg403        self._setuptools_cfg = setuptools_cfg404 405    def __enter__(self) -> "Self":406        """When entering the context, the values of ``packages``, ``py_modules`` and407        ``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``.408        """409        dist, cfg = self._dist, self._setuptools_cfg410        package_dir: Dict[str, str] = cfg.setdefault("package-dir", {})411        package_dir.update(dist.package_dir or {})412        dist.package_dir = package_dir  # needs to be the same object413 414        dist.set_defaults._ignore_ext_modules()  # pyproject.toml-specific behaviour415 416        # Set `name`, `py_modules` and `packages` in dist to short-circuit417        # auto-discovery, but avoid overwriting empty lists purposefully set by users.418        if dist.metadata.name is None:419            dist.metadata.name = self._project_cfg.get("name")420        if dist.py_modules is None:421            dist.py_modules = cfg.get("py-modules")422        if dist.packages is None:423            dist.packages = cfg.get("packages")424 425        return super().__enter__()426 427    def __exit__(self, exc_type, exc_value, traceback):428        """When exiting the context, if values of ``packages``, ``py_modules`` and429        ``package_dir`` are missing in ``setuptools_cfg``, copy from ``dist``.430        """431        # If anything was discovered set them back, so they count in the final config.432        self._setuptools_cfg.setdefault("packages", self._dist.packages)433        self._setuptools_cfg.setdefault("py-modules", self._dist.py_modules)434        return super().__exit__(exc_type, exc_value, traceback)435 436 437class _ExperimentalConfiguration(SetuptoolsWarning):438    _SUMMARY = (439        "`{subject}` in `pyproject.toml` is still *experimental* "440        "and likely to change in future releases."441    )442