codekingpro/portable-devtools

1"""Translation layer between pyproject config and setuptools distribution and2metadata objects.3 4The distribution and metadata objects are modeled after (an old version of)5core metadata, therefore configs in the format specified for ``pyproject.toml``6need to be processed before being applied.7 8**PRIVATE MODULE**: API reserved for setuptools internal usage only.9"""10 11from __future__ import annotations12 13import logging14import os15from collections.abc import Mapping16from email.headerregistry import Address17from functools import partial, reduce18from inspect import cleandoc19from itertools import chain20from types import MappingProxyType21from typing import TYPE_CHECKING, Any, Callable, TypeVar, Union22 23from .. import _static24from .._path import StrPath25from ..errors import InvalidConfigError, RemovedConfigError26from ..extension import Extension27from ..warnings import SetuptoolsDeprecationWarning, SetuptoolsWarning28 29if TYPE_CHECKING:30    from typing_extensions import TypeAlias31 32    from setuptools._importlib import metadata33    from setuptools.dist import Distribution34 35    from distutils.dist import _OptionsList  # Comes from typeshed36 37 38EMPTY: Mapping = MappingProxyType({})  # Immutable dict-like39_ProjectReadmeValue: TypeAlias = Union[str, dict[str, str]]40_Correspondence: TypeAlias = Callable[["Distribution", Any, Union[StrPath, None]], None]41_T = TypeVar("_T")42 43_logger = logging.getLogger(__name__)44 45 46def apply(dist: Distribution, config: dict, filename: StrPath) -> Distribution:47    """Apply configuration dict read with :func:`read_configuration`"""48 49    if not config:50        return dist  # short-circuit unrelated pyproject.toml file51 52    root_dir = os.path.dirname(filename) or "."53 54    _apply_project_table(dist, config, root_dir)55    _apply_tool_table(dist, config, filename)56 57    current_directory = os.getcwd()58    os.chdir(root_dir)59    try:60        dist._finalize_requires()61        dist._finalize_license_expression()62        dist._finalize_license_files()63    finally:64        os.chdir(current_directory)65 66    return dist67 68 69def _apply_project_table(dist: Distribution, config: dict, root_dir: StrPath):70    orig_config = config.get("project", {})71    if not orig_config:72        return  # short-circuit73 74    project_table = {k: _static.attempt_conversion(v) for k, v in orig_config.items()}75    _handle_missing_dynamic(dist, project_table)76    _unify_entry_points(project_table)77 78    for field, value in project_table.items():79        norm_key = json_compatible_key(field)80        corresp = PYPROJECT_CORRESPONDENCE.get(norm_key, norm_key)81        if callable(corresp):82            corresp(dist, value, root_dir)83        else:84            _set_config(dist, corresp, value)85 86 87def _apply_tool_table(dist: Distribution, config: dict, filename: StrPath):88    tool_table = config.get("tool", {}).get("setuptools", {})89    if not tool_table:90        return  # short-circuit91 92    if "license-files" in tool_table:93        if "license-files" in config.get("project", {}):94            # https://github.com/pypa/setuptools/pull/4837#discussion_r200498334995            raise InvalidConfigError(96                "'project.license-files' is defined already. "97                "Remove 'tool.setuptools.license-files'."98            )99 100        pypa_guides = "guides/writing-pyproject-toml/#license-files"101        SetuptoolsDeprecationWarning.emit(102            "'tool.setuptools.license-files' is deprecated in favor of "103            "'project.license-files' (available on setuptools>=77.0.0).",104            see_url=f"https://packaging.python.org/en/latest/{pypa_guides}",105            due_date=(2027, 2, 18),  # Warning introduced on 2025-02-18106        )107 108    for field, value in tool_table.items():109        norm_key = json_compatible_key(field)110 111        if norm_key in TOOL_TABLE_REMOVALS:112            suggestion = cleandoc(TOOL_TABLE_REMOVALS[norm_key])113            msg = f"""114            The parameter `tool.setuptools.{field}` was long deprecated115            and has been removed from `pyproject.toml`.116            """117            raise RemovedConfigError("\n".join([cleandoc(msg), suggestion]))118 119        norm_key = TOOL_TABLE_RENAMES.get(norm_key, norm_key)120        corresp = TOOL_TABLE_CORRESPONDENCE.get(norm_key, norm_key)121        if callable(corresp):122            corresp(dist, value)123        else:124            _set_config(dist, corresp, value)125 126    _copy_command_options(config, dist, filename)127 128 129def _handle_missing_dynamic(dist: Distribution, project_table: dict):130    """Be temporarily forgiving with ``dynamic`` fields not listed in ``dynamic``"""131    dynamic = set(project_table.get("dynamic", []))132    for field, getter in _PREVIOUSLY_DEFINED.items():133        if not (field in project_table or field in dynamic):134            value = getter(dist)135            if value:136                _MissingDynamic.emit(field=field, value=value)137                project_table[field] = _RESET_PREVIOUSLY_DEFINED.get(field)138 139 140def json_compatible_key(key: str) -> str:141    """As defined in :pep:`566#json-compatible-metadata`"""142    return key.lower().replace("-", "_")143 144 145def _set_config(dist: Distribution, field: str, value: Any):146    val = _PREPROCESS.get(field, _noop)(dist, value)147    setter = getattr(dist.metadata, f"set_{field}", None)148    if setter:149        setter(val)150    elif hasattr(dist.metadata, field) or field in SETUPTOOLS_PATCHES:151        setattr(dist.metadata, field, val)152    else:153        setattr(dist, field, val)154 155 156_CONTENT_TYPES = {157    ".md": "text/markdown",158    ".rst": "text/x-rst",159    ".txt": "text/plain",160}161 162 163def _guess_content_type(file: str) -> str | None:164    _, ext = os.path.splitext(file.lower())165    if not ext:166        return None167 168    if ext in _CONTENT_TYPES:169        return _static.Str(_CONTENT_TYPES[ext])170 171    valid = ", ".join(f"{k} ({v})" for k, v in _CONTENT_TYPES.items())172    msg = f"only the following file extensions are recognized: {valid}."173    raise ValueError(f"Undefined content type for {file}, {msg}")174 175 176def _long_description(177    dist: Distribution, val: _ProjectReadmeValue, root_dir: StrPath | None178):179    from setuptools.config import expand180 181    file: str | tuple[()]182    if isinstance(val, str):183        file = val184        text = expand.read_files(file, root_dir)185        ctype = _guess_content_type(file)186    else:187        file = val.get("file") or ()188        text = val.get("text") or expand.read_files(file, root_dir)189        ctype = val["content-type"]190 191    # XXX: Is it completely safe to assume static?192    _set_config(dist, "long_description", _static.Str(text))193 194    if ctype:195        _set_config(dist, "long_description_content_type", _static.Str(ctype))196 197    if file:198        dist._referenced_files.add(file)199 200 201def _license(dist: Distribution, val: str | dict, root_dir: StrPath | None):202    from setuptools.config import expand203 204    if isinstance(val, str):205        if getattr(dist.metadata, "license", None):206            SetuptoolsWarning.emit("`license` overwritten by `pyproject.toml`")207            dist.metadata.license = None208        _set_config(dist, "license_expression", _static.Str(val))209    else:210        pypa_guides = "guides/writing-pyproject-toml/#license"211        SetuptoolsDeprecationWarning.emit(212            "`project.license` as a TOML table is deprecated",213            "Please use a simple string containing a SPDX expression for "214            "`project.license`. You can also use `project.license-files`. "215            "(Both options available on setuptools>=77.0.0).",216            see_url=f"https://packaging.python.org/en/latest/{pypa_guides}",217            due_date=(2027, 2, 18),  # Introduced on 2025-02-18218        )219        if "file" in val:220            # XXX: Is it completely safe to assume static?221            value = expand.read_files([val["file"]], root_dir)222            _set_config(dist, "license", _static.Str(value))223            dist._referenced_files.add(val["file"])224        else:225            _set_config(dist, "license", _static.Str(val["text"]))226 227 228def _people(dist: Distribution, val: list[dict], _root_dir: StrPath | None, kind: str):229    field = []230    email_field = []231    for person in val:232        if "name" not in person:233            email_field.append(person["email"])234        elif "email" not in person:235            field.append(person["name"])236        else:237            addr = Address(display_name=person["name"], addr_spec=person["email"])238            email_field.append(str(addr))239 240    if field:241        _set_config(dist, kind, _static.Str(", ".join(field)))242    if email_field:243        _set_config(dist, f"{kind}_email", _static.Str(", ".join(email_field)))244 245 246def _project_urls(dist: Distribution, val: dict, _root_dir: StrPath | None):247    _set_config(dist, "project_urls", val)248 249 250def _python_requires(dist: Distribution, val: str, _root_dir: StrPath | None):251    _set_config(dist, "python_requires", _static.SpecifierSet(val))252 253 254def _dependencies(dist: Distribution, val: list, _root_dir: StrPath | None):255    if getattr(dist, "install_requires", []):256        msg = "`install_requires` overwritten in `pyproject.toml` (dependencies)"257        SetuptoolsWarning.emit(msg)258    dist.install_requires = val259 260 261def _optional_dependencies(dist: Distribution, val: dict, _root_dir: StrPath | None):262    if getattr(dist, "extras_require", None):263        msg = "`extras_require` overwritten in `pyproject.toml` (optional-dependencies)"264        SetuptoolsWarning.emit(msg)265    dist.extras_require = val266 267 268def _ext_modules(dist: Distribution, val: list[dict]) -> list[Extension]:269    existing = dist.ext_modules or []270    args = ({k.replace("-", "_"): v for k, v in x.items()} for x in val)271    new = (Extension(**_adjust_ext_attrs(kw)) for kw in args)272    return [*existing, *new]273 274 275def _adjust_ext_attrs(attrs: dict) -> dict:276    # https://github.com/pypa/setuptools/issues/4810277    # In TOML there is no differentiation between tuples and lists,278    # and distutils requires tuples...279    attrs["define_macros"] = list(map(tuple, attrs.get("define_macros") or []))280    return attrs281 282 283def _noop(_dist: Distribution, val: _T) -> _T:284    return val285 286 287def _identity(val: _T) -> _T:288    return val289 290 291def _unify_entry_points(project_table: dict):292    project = project_table293    given = project.pop("entry-points", project.pop("entry_points", {}))294    entry_points = dict(given)  # Avoid problems with static295    renaming = {"scripts": "console_scripts", "gui_scripts": "gui_scripts"}296    for key, value in list(project.items()):  # eager to allow modifications297        norm_key = json_compatible_key(key)298        if norm_key in renaming:299            # Don't skip even if value is empty (reason: reset missing `dynamic`)300            entry_points[renaming[norm_key]] = project.pop(key)301 302    if entry_points:303        project["entry-points"] = {304            name: [f"{k} = {v}" for k, v in group.items()]305            for name, group in entry_points.items()306            if group  # now we can skip empty groups307        }308        # Sometimes this will set `project["entry-points"] = {}`, and that is309        # intentional (for resetting configurations that are missing `dynamic`).310 311 312def _copy_command_options(pyproject: dict, dist: Distribution, filename: StrPath):313    tool_table = pyproject.get("tool", {})314    cmdclass = tool_table.get("setuptools", {}).get("cmdclass", {})315    valid_options = _valid_command_options(cmdclass)316 317    cmd_opts = dist.command_options318    for cmd, config in pyproject.get("tool", {}).get("distutils", {}).items():319        cmd = json_compatible_key(cmd)320        valid = valid_options.get(cmd, set())321        cmd_opts.setdefault(cmd, {})322        for key, value in config.items():323            key = json_compatible_key(key)324            cmd_opts[cmd][key] = (str(filename), value)325            if key not in valid:326                # To avoid removing options that are specified dynamically we327                # just log a warn...328                _logger.warning(f"Command option {cmd}.{key} is not defined")329 330 331def _valid_command_options(cmdclass: Mapping = EMPTY) -> dict[str, set[str]]:332    from setuptools.dist import Distribution333 334    from .._importlib import metadata335 336    valid_options = {"global": _normalise_cmd_options(Distribution.global_options)}337 338    unloaded_entry_points = metadata.entry_points(group='distutils.commands')339    loaded_entry_points = (_load_ep(ep) for ep in unloaded_entry_points)340    entry_points = (ep for ep in loaded_entry_points if ep)341    for cmd, cmd_class in chain(entry_points, cmdclass.items()):342        opts = valid_options.get(cmd, set())343        opts = opts | _normalise_cmd_options(getattr(cmd_class, "user_options", []))344        valid_options[cmd] = opts345 346    return valid_options347 348 349def _load_ep(ep: metadata.EntryPoint) -> tuple[str, type] | None:350    if ep.value.startswith("wheel.bdist_wheel"):351        # Ignore deprecated entrypoint from wheel and avoid warning pypa/wheel#631352        # TODO: remove check when `bdist_wheel` has been fully removed from pypa/wheel353        return None354 355    # Ignore all the errors356    try:357        return (ep.name, ep.load())358    except Exception as ex:359        msg = f"{ex.__class__.__name__} while trying to load entry-point {ep.name}"360        _logger.warning(f"{msg}: {ex}")361        return None362 363 364def _normalise_cmd_option_key(name: str) -> str:365    return json_compatible_key(name).strip("_=")366 367 368def _normalise_cmd_options(desc: _OptionsList) -> set[str]:369    return {_normalise_cmd_option_key(fancy_option[0]) for fancy_option in desc}370 371 372def _get_previous_entrypoints(dist: Distribution) -> dict[str, list]:373    ignore = ("console_scripts", "gui_scripts")374    value = getattr(dist, "entry_points", None) or {}375    return {k: v for k, v in value.items() if k not in ignore}376 377 378def _get_previous_scripts(dist: Distribution) -> list | None:379    value = getattr(dist, "entry_points", None) or {}380    return value.get("console_scripts")381 382 383def _get_previous_gui_scripts(dist: Distribution) -> list | None:384    value = getattr(dist, "entry_points", None) or {}385    return value.get("gui_scripts")386 387 388def _set_static_list_metadata(attr: str, dist: Distribution, val: list) -> None:389    """Apply distutils metadata validation but preserve "static" behaviour"""390    meta = dist.metadata391    setter, getter = getattr(meta, f"set_{attr}"), getattr(meta, f"get_{attr}")392    setter(val)393    setattr(meta, attr, _static.List(getter()))394 395 396def _attrgetter(attr):397    """398    Similar to ``operator.attrgetter`` but returns None if ``attr`` is not found399    >>> from types import SimpleNamespace400    >>> obj = SimpleNamespace(a=42, b=SimpleNamespace(c=13))401    >>> _attrgetter("a")(obj)402    42403    >>> _attrgetter("b.c")(obj)404    13405    >>> _attrgetter("d")(obj) is None406    True407    """408    return partial(reduce, lambda acc, x: getattr(acc, x, None), attr.split("."))409 410 411def _some_attrgetter(*items):412    """413    Return the first "truth-y" attribute or None414    >>> from types import SimpleNamespace415    >>> obj = SimpleNamespace(a=42, b=SimpleNamespace(c=13))416    >>> _some_attrgetter("d", "a", "b.c")(obj)417    42418    >>> _some_attrgetter("d", "e", "b.c", "a")(obj)419    13420    >>> _some_attrgetter("d", "e", "f")(obj) is None421    True422    """423 424    def _acessor(obj):425        values = (_attrgetter(i)(obj) for i in items)426        return next((i for i in values if i is not None), None)427 428    return _acessor429 430 431PYPROJECT_CORRESPONDENCE: dict[str, _Correspondence] = {432    "readme": _long_description,433    "license": _license,434    "authors": partial(_people, kind="author"),435    "maintainers": partial(_people, kind="maintainer"),436    "urls": _project_urls,437    "dependencies": _dependencies,438    "optional_dependencies": _optional_dependencies,439    "requires_python": _python_requires,440}441 442TOOL_TABLE_RENAMES = {"script_files": "scripts"}443TOOL_TABLE_REMOVALS = {444    "namespace_packages": """445        Please migrate to implicit native namespaces instead.446        See https://packaging.python.org/en/latest/guides/packaging-namespace-packages/.447        """,448}449TOOL_TABLE_CORRESPONDENCE = {450    # Fields with corresponding core metadata need to be marked as static:451    "obsoletes": partial(_set_static_list_metadata, "obsoletes"),452    "provides": partial(_set_static_list_metadata, "provides"),453    "platforms": partial(_set_static_list_metadata, "platforms"),454}455 456SETUPTOOLS_PATCHES = {457    "long_description_content_type",458    "project_urls",459    "provides_extras",460    "license_file",461    "license_files",462    "license_expression",463}464 465_PREPROCESS = {466    "ext_modules": _ext_modules,467}468 469_PREVIOUSLY_DEFINED = {470    "name": _attrgetter("metadata.name"),471    "version": _attrgetter("metadata.version"),472    "description": _attrgetter("metadata.description"),473    "readme": _attrgetter("metadata.long_description"),474    "requires-python": _some_attrgetter("python_requires", "metadata.python_requires"),475    "license": _some_attrgetter("metadata.license_expression", "metadata.license"),476    # XXX: `license-file` is currently not considered in the context of `dynamic`.477    #      See TestPresetField.test_license_files_exempt_from_dynamic478    "authors": _some_attrgetter("metadata.author", "metadata.author_email"),479    "maintainers": _some_attrgetter("metadata.maintainer", "metadata.maintainer_email"),480    "keywords": _attrgetter("metadata.keywords"),481    "classifiers": _attrgetter("metadata.classifiers"),482    "urls": _attrgetter("metadata.project_urls"),483    "entry-points": _get_previous_entrypoints,484    "scripts": _get_previous_scripts,485    "gui-scripts": _get_previous_gui_scripts,486    "dependencies": _attrgetter("install_requires"),487    "optional-dependencies": _attrgetter("extras_require"),488}489 490 491_RESET_PREVIOUSLY_DEFINED: dict = {492    # Fix improper setting: given in `setup.py`, but not listed in `dynamic`493    # Use "immutable" data structures to avoid in-place modification.494    # dict: pyproject name => value to which reset495    "license": "",496    # XXX: `license-file` is currently not considered in the context of `dynamic`.497    #      See TestPresetField.test_license_files_exempt_from_dynamic498    "authors": _static.EMPTY_LIST,499    "maintainers": _static.EMPTY_LIST,500    "keywords": _static.EMPTY_LIST,501    "classifiers": _static.EMPTY_LIST,502    "urls": _static.EMPTY_DICT,503    "entry-points": _static.EMPTY_DICT,504    "scripts": _static.EMPTY_DICT,505    "gui-scripts": _static.EMPTY_DICT,506    "dependencies": _static.EMPTY_LIST,507    "optional-dependencies": _static.EMPTY_DICT,508}509 510 511class _MissingDynamic(SetuptoolsWarning):512    _SUMMARY = "`{field}` defined outside of `pyproject.toml` is ignored."513 514    _DETAILS = """515    The following seems to be defined outside of `pyproject.toml`:516 517    `{field} = {value!r}`518 519    According to the spec (see the link below), however, setuptools CANNOT520    consider this value unless `{field}` is listed as `dynamic`.521 522    https://packaging.python.org/en/latest/specifications/pyproject-toml/#declaring-project-metadata-the-project-table523 524    To prevent this problem, you can list `{field}` under `dynamic` or alternatively525    remove the `[project]` table from your file and rely entirely on other means of526    configuration.527    """528    # TODO: Consider removing this check in the future?529    #       There is a trade-off here between improving "debug-ability" and the cost530    #       of running/testing/maintaining these unnecessary checks...531 532    @classmethod533    def details(cls, field: str, value: Any) -> str:534        return cls._DETAILS.format(field=field, value=value)535