codekingpro/portable-devtools

1"""2Load setuptools configuration from ``setup.cfg`` files.3 4**API will be made private in the future**5 6To read project metadata, consider using7``build.util.project_wheel_metadata`` (https://pypi.org/project/build/).8For simple scenarios, you can also try parsing the file directly9with the help of ``configparser``.10"""11 12from __future__ import annotations13 14import contextlib15import functools16import os17from abc import abstractmethod18from collections import defaultdict19from collections.abc import Iterable, Iterator20from functools import partial, wraps21from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, TypeVar, cast22 23from packaging.markers import default_environment as marker_env24from packaging.requirements import InvalidRequirement, Requirement25from packaging.version import InvalidVersion, Version26 27from .. import _static28from .._path import StrPath29from ..errors import FileError, OptionError30from ..warnings import SetuptoolsDeprecationWarning31from . import expand32 33if TYPE_CHECKING:34    from typing_extensions import TypeAlias35 36    from setuptools.dist import Distribution37 38    from distutils.dist import DistributionMetadata39 40SingleCommandOptions: TypeAlias = dict[str, tuple[str, Any]]41"""Dict that associate the name of the options of a particular command to a42tuple. The first element of the tuple indicates the origin of the option value43(e.g. the name of the configuration file where it was read from),44while the second element of the tuple is the option value itself45"""46AllCommandOptions: TypeAlias = dict[str, SingleCommandOptions]47"""cmd name => its options"""48Target = TypeVar("Target", "Distribution", "DistributionMetadata")49 50 51def read_configuration(52    filepath: StrPath, find_others: bool = False, ignore_option_errors: bool = False53) -> dict:54    """Read given configuration file and returns options from it as a dict.55 56    :param str|unicode filepath: Path to configuration file57        to get options from.58 59    :param bool find_others: Whether to search for other configuration files60        which could be on in various places.61 62    :param bool ignore_option_errors: Whether to silently ignore63        options, values of which could not be resolved (e.g. due to exceptions64        in directives such as file:, attr:, etc.).65        If False exceptions are propagated as expected.66 67    :rtype: dict68    """69    from setuptools.dist import Distribution70 71    dist = Distribution()72    filenames = dist.find_config_files() if find_others else []73    handlers = _apply(dist, filepath, filenames, ignore_option_errors)74    return configuration_to_dict(handlers)75 76 77def apply_configuration(dist: Distribution, filepath: StrPath) -> Distribution:78    """Apply the configuration from a ``setup.cfg`` file into an existing79    distribution object.80    """81    _apply(dist, filepath)82    dist._finalize_requires()83    return dist84 85 86def _apply(87    dist: Distribution,88    filepath: StrPath,89    other_files: Iterable[StrPath] = (),90    ignore_option_errors: bool = False,91) -> tuple[ConfigMetadataHandler, ConfigOptionsHandler]:92    """Read configuration from ``filepath`` and applies to the ``dist`` object."""93    from setuptools.dist import _Distribution94 95    filepath = os.path.abspath(filepath)96 97    if not os.path.isfile(filepath):98        raise FileError(f'Configuration file {filepath} does not exist.')99 100    current_directory = os.getcwd()101    os.chdir(os.path.dirname(filepath))102    filenames = [*other_files, filepath]103 104    try:105        # TODO: Temporary cast until mypy 1.12 is released with upstream fixes from typeshed106        _Distribution.parse_config_files(dist, filenames=cast(list[str], filenames))107        handlers = parse_configuration(108            dist, dist.command_options, ignore_option_errors=ignore_option_errors109        )110        dist._finalize_license_files()111    finally:112        os.chdir(current_directory)113 114    return handlers115 116 117def _get_option(target_obj: Distribution | DistributionMetadata, key: str):118    """119    Given a target object and option key, get that option from120    the target object, either through a get_{key} method or121    from an attribute directly.122    """123    getter_name = f'get_{key}'124    by_attribute = functools.partial(getattr, target_obj, key)125    getter = getattr(target_obj, getter_name, by_attribute)126    return getter()127 128 129def configuration_to_dict(130    handlers: Iterable[131        ConfigHandler[Distribution] | ConfigHandler[DistributionMetadata]132    ],133) -> dict:134    """Returns configuration data gathered by given handlers as a dict.135 136    :param Iterable[ConfigHandler] handlers: Handlers list,137        usually from parse_configuration()138 139    :rtype: dict140    """141    config_dict: dict = defaultdict(dict)142 143    for handler in handlers:144        for option in handler.set_options:145            value = _get_option(handler.target_obj, option)146            config_dict[handler.section_prefix][option] = value147 148    return config_dict149 150 151def parse_configuration(152    distribution: Distribution,153    command_options: AllCommandOptions,154    ignore_option_errors: bool = False,155) -> tuple[ConfigMetadataHandler, ConfigOptionsHandler]:156    """Performs additional parsing of configuration options157    for a distribution.158 159    Returns a list of used option handlers.160 161    :param Distribution distribution:162    :param dict command_options:163    :param bool ignore_option_errors: Whether to silently ignore164        options, values of which could not be resolved (e.g. due to exceptions165        in directives such as file:, attr:, etc.).166        If False exceptions are propagated as expected.167    :rtype: list168    """169    with expand.EnsurePackagesDiscovered(distribution) as ensure_discovered:170        options = ConfigOptionsHandler(171            distribution,172            command_options,173            ignore_option_errors,174            ensure_discovered,175        )176 177        options.parse()178        if not distribution.package_dir:179            distribution.package_dir = options.package_dir  # Filled by `find_packages`180 181        meta = ConfigMetadataHandler(182            distribution.metadata,183            command_options,184            ignore_option_errors,185            ensure_discovered,186            distribution.package_dir,187            distribution.src_root,188        )189        meta.parse()190        distribution._referenced_files.update(191            options._referenced_files, meta._referenced_files192        )193 194    return meta, options195 196 197def _warn_accidental_env_marker_misconfig(label: str, orig_value: str, parsed: list):198    """Because users sometimes misinterpret this configuration:199 200    [options.extras_require]201    foo = bar;python_version<"4"202 203    It looks like one requirement with an environment marker204    but because there is no newline, it's parsed as two requirements205    with a semicolon as separator.206 207    Therefore, if:208        * input string does not contain a newline AND209        * parsed result contains two requirements AND210        * parsing of the two parts from the result ("<first>;<second>")211        leads in a valid Requirement with a valid marker212    a UserWarning is shown to inform the user about the possible problem.213    """214    if "\n" in orig_value or len(parsed) != 2:215        return216 217    markers = marker_env().keys()218 219    try:220        req = Requirement(parsed[1])221        if req.name in markers:222            _AmbiguousMarker.emit(field=label, req=parsed[1])223    except InvalidRequirement as ex:224        if any(parsed[1].startswith(marker) for marker in markers):225            msg = _AmbiguousMarker.message(field=label, req=parsed[1])226            raise InvalidRequirement(msg) from ex227 228 229class ConfigHandler(Generic[Target]):230    """Handles metadata supplied in configuration files."""231 232    section_prefix: str233    """Prefix for config sections handled by this handler.234    Must be provided by class heirs.235 236    """237 238    aliases: ClassVar[dict[str, str]] = {}239    """Options aliases.240    For compatibility with various packages. E.g.: d2to1 and pbr.241    Note: `-` in keys is replaced with `_` by config parser.242 243    """244 245    def __init__(246        self,247        target_obj: Target,248        options: AllCommandOptions,249        ignore_option_errors,250        ensure_discovered: expand.EnsurePackagesDiscovered,251    ) -> None:252        self.ignore_option_errors = ignore_option_errors253        self.target_obj: Target = target_obj254        self.sections = dict(self._section_options(options))255        self.set_options: list[str] = []256        self.ensure_discovered = ensure_discovered257        self._referenced_files = set[str]()258        """After parsing configurations, this property will enumerate259        all files referenced by the "file:" directive. Private API for setuptools only.260        """261 262    @classmethod263    def _section_options(264        cls, options: AllCommandOptions265    ) -> Iterator[tuple[str, SingleCommandOptions]]:266        for full_name, value in options.items():267            pre, _sep, name = full_name.partition(cls.section_prefix)268            if pre:269                continue270            yield name.lstrip('.'), value271 272    @property273    @abstractmethod274    def parsers(self) -> dict[str, Callable]:275        """Metadata item name to parser function mapping."""276        raise NotImplementedError(277            f'{self.__class__.__name__} must provide .parsers property'278        )279 280    def __setitem__(self, option_name, value) -> None:281        target_obj = self.target_obj282 283        # Translate alias into real name.284        option_name = self.aliases.get(option_name, option_name)285 286        try:287            current_value = getattr(target_obj, option_name)288        except AttributeError as e:289            raise KeyError(option_name) from e290 291        if current_value:292            # Already inhabited. Skipping.293            return294 295        try:296            parsed = self.parsers.get(option_name, lambda x: x)(value)297        except (Exception,) * self.ignore_option_errors:298            return299 300        simple_setter = functools.partial(target_obj.__setattr__, option_name)301        setter = getattr(target_obj, f"set_{option_name}", simple_setter)302        setter(parsed)303 304        self.set_options.append(option_name)305 306    @classmethod307    def _parse_list(cls, value, separator=','):308        """Represents value as a list.309 310        Value is split either by separator (defaults to comma) or by lines.311 312        :param value:313        :param separator: List items separator character.314        :rtype: list315        """316        if isinstance(value, list):  # _get_parser_compound case317            return value318 319        if '\n' in value:320            value = value.splitlines()321        else:322            value = value.split(separator)323 324        return [chunk.strip() for chunk in value if chunk.strip()]325 326    @classmethod327    def _parse_dict(cls, value):328        """Represents value as a dict.329 330        :param value:331        :rtype: dict332        """333        separator = '='334        result = {}335        for line in cls._parse_list(value):336            key, sep, val = line.partition(separator)337            if sep != separator:338                raise OptionError(f"Unable to parse option value to dict: {value}")339            result[key.strip()] = val.strip()340 341        return result342 343    @classmethod344    def _parse_bool(cls, value):345        """Represents value as boolean.346 347        :param value:348        :rtype: bool349        """350        value = value.lower()351        return value in ('1', 'true', 'yes')352 353    @classmethod354    def _exclude_files_parser(cls, key):355        """Returns a parser function to make sure field inputs356        are not files.357 358        Parses a value after getting the key so error messages are359        more informative.360 361        :param key:362        :rtype: callable363        """364 365        def parser(value):366            exclude_directive = 'file:'367            if value.startswith(exclude_directive):368                raise ValueError(369                    f'Only strings are accepted for the {key} field, '370                    'files are not accepted'371                )372            return _static.Str(value)373 374        return parser375 376    def _parse_file(self, value, root_dir: StrPath | None):377        """Represents value as a string, allowing including text378        from nearest files using `file:` directive.379 380        Directive is sandboxed and won't reach anything outside381        directory with setup.py.382 383        Examples:384            file: README.rst, CHANGELOG.md, src/file.txt385 386        :param str value:387        :rtype: str388        """389        include_directive = 'file:'390 391        if not isinstance(value, str):392            return value393 394        if not value.startswith(include_directive):395            return _static.Str(value)396 397        spec = value[len(include_directive) :]398        filepaths = [path.strip() for path in spec.split(',')]399        self._referenced_files.update(filepaths)400        # XXX: Is marking as static contents coming from files too optimistic?401        return _static.Str(expand.read_files(filepaths, root_dir))402 403    def _parse_attr(self, value, package_dir, root_dir: StrPath):404        """Represents value as a module attribute.405 406        Examples:407            attr: package.attr408            attr: package.module.attr409 410        :param str value:411        :rtype: str412        """413        attr_directive = 'attr:'414        if not value.startswith(attr_directive):415            return _static.Str(value)416 417        attr_desc = value.replace(attr_directive, '')418 419        # Make sure package_dir is populated correctly, so `attr:` directives can work420        package_dir.update(self.ensure_discovered.package_dir)421        return expand.read_attr(attr_desc, package_dir, root_dir)422 423    @classmethod424    def _get_parser_compound(cls, *parse_methods):425        """Returns parser function to represents value as a list.426 427        Parses a value applying given methods one after another.428 429        :param parse_methods:430        :rtype: callable431        """432 433        def parse(value):434            parsed = value435 436            for method in parse_methods:437                parsed = method(parsed)438 439            return parsed440 441        return parse442 443    @classmethod444    def _parse_section_to_dict_with_key(cls, section_options, values_parser):445        """Parses section options into a dictionary.446 447        Applies a given parser to each option in a section.448 449        :param dict section_options:450        :param callable values_parser: function with 2 args corresponding to key, value451        :rtype: dict452        """453        value = {}454        for key, (_, val) in section_options.items():455            value[key] = values_parser(key, val)456        return value457 458    @classmethod459    def _parse_section_to_dict(cls, section_options, values_parser=None):460        """Parses section options into a dictionary.461 462        Optionally applies a given parser to each value.463 464        :param dict section_options:465        :param callable values_parser: function with 1 arg corresponding to option value466        :rtype: dict467        """468        parser = (lambda _, v: values_parser(v)) if values_parser else (lambda _, v: v)469        return cls._parse_section_to_dict_with_key(section_options, parser)470 471    def parse_section(self, section_options) -> None:472        """Parses configuration file section.473 474        :param dict section_options:475        """476        for name, (_, value) in section_options.items():477            with contextlib.suppress(KeyError):478                # Keep silent for a new option may appear anytime.479                self[name] = value480 481    def parse(self) -> None:482        """Parses configuration file items from one483        or more related sections.484 485        """486        for section_name, section_options in self.sections.items():487            method_postfix = ''488            if section_name:  # [section.option] variant489                method_postfix = f"_{section_name}"490 491            section_parser_method: Callable | None = getattr(492                self,493                # Dots in section names are translated into dunderscores.494                f'parse_section{method_postfix}'.replace('.', '__'),495                None,496            )497 498            if section_parser_method is None:499                raise OptionError(500                    "Unsupported distribution option section: "501                    f"[{self.section_prefix}.{section_name}]"502                )503 504            section_parser_method(section_options)505 506    def _deprecated_config_handler(self, func, msg, **kw):507        """this function will wrap around parameters that are deprecated508 509        :param msg: deprecation message510        :param func: function to be wrapped around511        """512 513        @wraps(func)514        def config_handler(*args, **kwargs):515            kw.setdefault("stacklevel", 2)516            _DeprecatedConfig.emit("Deprecated config in `setup.cfg`", msg, **kw)517            return func(*args, **kwargs)518 519        return config_handler520 521 522class ConfigMetadataHandler(ConfigHandler["DistributionMetadata"]):523    section_prefix = 'metadata'524 525    aliases = {526        'home_page': 'url',527        'summary': 'description',528        'classifier': 'classifiers',529        'platform': 'platforms',530    }531 532    strict_mode = False533    """We need to keep it loose, to be partially compatible with534    `pbr` and `d2to1` packages which also uses `metadata` section.535 536    """537 538    def __init__(539        self,540        target_obj: DistributionMetadata,541        options: AllCommandOptions,542        ignore_option_errors: bool,543        ensure_discovered: expand.EnsurePackagesDiscovered,544        package_dir: dict | None = None,545        root_dir: StrPath | None = os.curdir,546    ) -> None:547        super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)548        self.package_dir = package_dir549        self.root_dir = root_dir550 551    @property552    def parsers(self) -> dict[str, Callable]:553        """Metadata item name to parser function mapping."""554        parse_list_static = self._get_parser_compound(self._parse_list, _static.List)555        parse_dict_static = self._get_parser_compound(self._parse_dict, _static.Dict)556        parse_file = partial(self._parse_file, root_dir=self.root_dir)557        exclude_files_parser = self._exclude_files_parser558 559        return {560            'author': _static.Str,561            'author_email': _static.Str,562            'maintainer': _static.Str,563            'maintainer_email': _static.Str,564            'platforms': parse_list_static,565            'keywords': parse_list_static,566            'provides': parse_list_static,567            'obsoletes': parse_list_static,568            'classifiers': self._get_parser_compound(parse_file, parse_list_static),569            'license': exclude_files_parser('license'),570            'license_files': parse_list_static,571            'description': parse_file,572            'long_description': parse_file,573            'long_description_content_type': _static.Str,574            'version': self._parse_version,  # Cannot be marked as dynamic575            'url': _static.Str,576            'project_urls': parse_dict_static,577        }578 579    def _parse_version(self, value):580        """Parses `version` option value.581 582        :param value:583        :rtype: str584 585        """586        version = self._parse_file(value, self.root_dir)587 588        if version != value:589            version = version.strip()590            # Be strict about versions loaded from file because it's easy to591            # accidentally include newlines and other unintended content592            try:593                Version(version)594            except InvalidVersion as e:595                raise OptionError(596                    f'Version loaded from {value} does not '597                    f'comply with PEP 440: {version}'598                ) from e599 600            return version601 602        return expand.version(self._parse_attr(value, self.package_dir, self.root_dir))603 604 605class ConfigOptionsHandler(ConfigHandler["Distribution"]):606    section_prefix = 'options'607 608    def __init__(609        self,610        target_obj: Distribution,611        options: AllCommandOptions,612        ignore_option_errors: bool,613        ensure_discovered: expand.EnsurePackagesDiscovered,614    ) -> None:615        super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)616        self.root_dir = target_obj.src_root617        self.package_dir: dict[str, str] = {}  # To be filled by `find_packages`618 619    @classmethod620    def _parse_list_semicolon(cls, value):621        return cls._parse_list(value, separator=';')622 623    def _parse_file_in_root(self, value):624        return self._parse_file(value, root_dir=self.root_dir)625 626    def _parse_requirements_list(self, label: str, value: str):627        # Parse a requirements list, either by reading in a `file:`, or a list.628        parsed = self._parse_list_semicolon(self._parse_file_in_root(value))629        _warn_accidental_env_marker_misconfig(label, value, parsed)630        # Filter it to only include lines that are not comments. `parse_list`631        # will have stripped each line and filtered out empties.632        return _static.List(line for line in parsed if not line.startswith("#"))633        # ^-- Use `_static.List` to mark a non-`Dynamic` Core Metadata634 635    @property636    def parsers(self) -> dict[str, Callable]:637        """Metadata item name to parser function mapping."""638        parse_list = self._parse_list639        parse_bool = self._parse_bool640        parse_cmdclass = self._parse_cmdclass641 642        return {643            'zip_safe': parse_bool,644            'include_package_data': parse_bool,645            'package_dir': self._parse_dict,646            'scripts': parse_list,647            'eager_resources': parse_list,648            'dependency_links': parse_list,649            'namespace_packages': self._deprecated_config_handler(650                parse_list,651                "The namespace_packages parameter is deprecated, "652                "consider using implicit namespaces instead (PEP 420).",653                # TODO: define due date, see setuptools.dist:check_nsp.654            ),655            'install_requires': partial(  # Core Metadata656                self._parse_requirements_list, "install_requires"657            ),658            'setup_requires': self._parse_list_semicolon,659            'packages': self._parse_packages,660            'entry_points': self._parse_file_in_root,661            'py_modules': parse_list,662            'python_requires': _static.SpecifierSet,  # Core Metadata663            'cmdclass': parse_cmdclass,664        }665 666    def _parse_cmdclass(self, value):667        package_dir = self.ensure_discovered.package_dir668        return expand.cmdclass(self._parse_dict(value), package_dir, self.root_dir)669 670    def _parse_packages(self, value):671        """Parses `packages` option value.672 673        :param value:674        :rtype: list675        """676        find_directives = ['find:', 'find_namespace:']677        trimmed_value = value.strip()678 679        if trimmed_value not in find_directives:680            return self._parse_list(value)681 682        # Read function arguments from a dedicated section.683        find_kwargs = self.parse_section_packages__find(684            self.sections.get('packages.find', {})685        )686 687        find_kwargs.update(688            namespaces=(trimmed_value == find_directives[1]),689            root_dir=self.root_dir,690            fill_package_dir=self.package_dir,691        )692 693        return expand.find_packages(**find_kwargs)694 695    def parse_section_packages__find(self, section_options):696        """Parses `packages.find` configuration file section.697 698        To be used in conjunction with _parse_packages().699 700        :param dict section_options:701        """702        section_data = self._parse_section_to_dict(section_options, self._parse_list)703 704        valid_keys = ['where', 'include', 'exclude']705        find_kwargs = {k: v for k, v in section_data.items() if k in valid_keys and v}706 707        where = find_kwargs.get('where')708        if where is not None:709            find_kwargs['where'] = where[0]  # cast list to single val710 711        return find_kwargs712 713    def parse_section_entry_points(self, section_options) -> None:714        """Parses `entry_points` configuration file section.715 716        :param dict section_options:717        """718        parsed = self._parse_section_to_dict(section_options, self._parse_list)719        self['entry_points'] = parsed720 721    def _parse_package_data(self, section_options):722        package_data = self._parse_section_to_dict(section_options, self._parse_list)723        return expand.canonic_package_data(package_data)724 725    def parse_section_package_data(self, section_options) -> None:726        """Parses `package_data` configuration file section.727 728        :param dict section_options:729        """730        self['package_data'] = self._parse_package_data(section_options)731 732    def parse_section_exclude_package_data(self, section_options) -> None:733        """Parses `exclude_package_data` configuration file section.734 735        :param dict section_options:736        """737        self['exclude_package_data'] = self._parse_package_data(section_options)738 739    def parse_section_extras_require(self, section_options) -> None:  # Core Metadata740        """Parses `extras_require` configuration file section.741 742        :param dict section_options:743        """744        parsed = self._parse_section_to_dict_with_key(745            section_options,746            lambda k, v: self._parse_requirements_list(f"extras_require[{k}]", v),747        )748 749        self['extras_require'] = _static.Dict(parsed)750        # ^-- Use `_static.Dict` to mark a non-`Dynamic` Core Metadata751 752    def parse_section_data_files(self, section_options) -> None:753        """Parses `data_files` configuration file section.754 755        :param dict section_options:756        """757        parsed = self._parse_section_to_dict(section_options, self._parse_list)758        self['data_files'] = expand.canonic_data_files(parsed, self.root_dir)759 760 761class _AmbiguousMarker(SetuptoolsDeprecationWarning):762    _SUMMARY = "Ambiguous requirement marker."763    _DETAILS = """764    One of the parsed requirements in `{field}` looks like a valid environment marker:765 766        {req!r}767 768    Please make sure that the configuration file is correct.769    You can use dangling lines to avoid this problem.770    """771    _SEE_DOCS = "userguide/declarative_config.html#opt-2"772    # TODO: should we include due_date here? Initially introduced in 6 Aug 2022.773    # Does this make sense with latest version of packaging?774 775    @classmethod776    def message(cls, **kw):777        docs = f"https://setuptools.pypa.io/en/latest/{cls._SEE_DOCS}"778        return cls._format(cls._SUMMARY, cls._DETAILS, see_url=docs, format_args=kw)779 780 781class _DeprecatedConfig(SetuptoolsDeprecationWarning):782    _SEE_DOCS = "userguide/declarative_config.html"783