codekingpro/portable-devtools

1"""2Load setuptools configuration from ``setup.cfg`` files.3 4**API will be made private in the future**5 6To read project metadata, consider using7``build.util.project_wheel_metadata`` (https://pypi.org/project/build/).8For simple scenarios, you can also try parsing the file directly9with the help of ``configparser``.10"""11 12import contextlib13import functools14import os15from collections import defaultdict16from functools import partial17from functools import wraps18from typing import (19    TYPE_CHECKING,20    Callable,21    Any,22    Dict,23    Generic,24    Iterable,25    List,26    Optional,27    Set,28    Tuple,29    TypeVar,30    Union,31)32 33from .._path import StrPath34from ..errors import FileError, OptionError35from ..extern.packaging.markers import default_environment as marker_env36from ..extern.packaging.requirements import InvalidRequirement, Requirement37from ..extern.packaging.specifiers import SpecifierSet38from ..extern.packaging.version import InvalidVersion, Version39from ..warnings import SetuptoolsDeprecationWarning40from . import expand41 42if TYPE_CHECKING:43    from distutils.dist import DistributionMetadata  # noqa44 45    from setuptools.dist import Distribution  # noqa46 47SingleCommandOptions = Dict["str", Tuple["str", Any]]48"""Dict that associate the name of the options of a particular command to a49tuple. The first element of the tuple indicates the origin of the option value50(e.g. the name of the configuration file where it was read from),51while the second element of the tuple is the option value itself52"""53AllCommandOptions = Dict["str", SingleCommandOptions]  # cmd name => its options54Target = TypeVar("Target", bound=Union["Distribution", "DistributionMetadata"])55 56 57def read_configuration(58    filepath: StrPath, find_others=False, ignore_option_errors=False59) -> dict:60    """Read given configuration file and returns options from it as a dict.61 62    :param str|unicode filepath: Path to configuration file63        to get options from.64 65    :param bool find_others: Whether to search for other configuration files66        which could be on in various places.67 68    :param bool ignore_option_errors: Whether to silently ignore69        options, values of which could not be resolved (e.g. due to exceptions70        in directives such as file:, attr:, etc.).71        If False exceptions are propagated as expected.72 73    :rtype: dict74    """75    from setuptools.dist import Distribution76 77    dist = Distribution()78    filenames = dist.find_config_files() if find_others else []79    handlers = _apply(dist, filepath, filenames, ignore_option_errors)80    return configuration_to_dict(handlers)81 82 83def apply_configuration(dist: "Distribution", filepath: StrPath) -> "Distribution":84    """Apply the configuration from a ``setup.cfg`` file into an existing85    distribution object.86    """87    _apply(dist, filepath)88    dist._finalize_requires()89    return dist90 91 92def _apply(93    dist: "Distribution",94    filepath: StrPath,95    other_files: Iterable[StrPath] = (),96    ignore_option_errors: bool = False,97) -> Tuple["ConfigHandler", ...]:98    """Read configuration from ``filepath`` and applies to the ``dist`` object."""99    from setuptools.dist import _Distribution100 101    filepath = os.path.abspath(filepath)102 103    if not os.path.isfile(filepath):104        raise FileError(f'Configuration file {filepath} does not exist.')105 106    current_directory = os.getcwd()107    os.chdir(os.path.dirname(filepath))108    filenames = [*other_files, filepath]109 110    try:111        _Distribution.parse_config_files(dist, filenames=filenames)  # type: ignore[arg-type] # TODO: fix in disutils stubs112        handlers = parse_configuration(113            dist, dist.command_options, ignore_option_errors=ignore_option_errors114        )115        dist._finalize_license_files()116    finally:117        os.chdir(current_directory)118 119    return handlers120 121 122def _get_option(target_obj: Target, key: str):123    """124    Given a target object and option key, get that option from125    the target object, either through a get_{key} method or126    from an attribute directly.127    """128    getter_name = f'get_{key}'129    by_attribute = functools.partial(getattr, target_obj, key)130    getter = getattr(target_obj, getter_name, by_attribute)131    return getter()132 133 134def configuration_to_dict(handlers: Tuple["ConfigHandler", ...]) -> dict:135    """Returns configuration data gathered by given handlers as a dict.136 137    :param list[ConfigHandler] handlers: Handlers list,138        usually from parse_configuration()139 140    :rtype: dict141    """142    config_dict: dict = defaultdict(dict)143 144    for handler in handlers:145        for option in handler.set_options:146            value = _get_option(handler.target_obj, option)147            config_dict[handler.section_prefix][option] = value148 149    return config_dict150 151 152def parse_configuration(153    distribution: "Distribution",154    command_options: AllCommandOptions,155    ignore_option_errors=False,156) -> Tuple["ConfigMetadataHandler", "ConfigOptionsHandler"]:157    """Performs additional parsing of configuration options158    for a distribution.159 160    Returns a list of used option handlers.161 162    :param Distribution distribution:163    :param dict command_options:164    :param bool ignore_option_errors: Whether to silently ignore165        options, values of which could not be resolved (e.g. due to exceptions166        in directives such as file:, attr:, etc.).167        If False exceptions are propagated as expected.168    :rtype: list169    """170    with expand.EnsurePackagesDiscovered(distribution) as ensure_discovered:171        options = ConfigOptionsHandler(172            distribution,173            command_options,174            ignore_option_errors,175            ensure_discovered,176        )177 178        options.parse()179        if not distribution.package_dir:180            distribution.package_dir = options.package_dir  # Filled by `find_packages`181 182        meta = ConfigMetadataHandler(183            distribution.metadata,184            command_options,185            ignore_option_errors,186            ensure_discovered,187            distribution.package_dir,188            distribution.src_root,189        )190        meta.parse()191        distribution._referenced_files.update(192            options._referenced_files, meta._referenced_files193        )194 195    return meta, options196 197 198def _warn_accidental_env_marker_misconfig(label: str, orig_value: str, parsed: list):199    """Because users sometimes misinterpret this configuration:200 201    [options.extras_require]202    foo = bar;python_version<"4"203 204    It looks like one requirement with an environment marker205    but because there is no newline, it's parsed as two requirements206    with a semicolon as separator.207 208    Therefore, if:209        * input string does not contain a newline AND210        * parsed result contains two requirements AND211        * parsing of the two parts from the result ("<first>;<second>")212        leads in a valid Requirement with a valid marker213    a UserWarning is shown to inform the user about the possible problem.214    """215    if "\n" in orig_value or len(parsed) != 2:216        return217 218    markers = marker_env().keys()219 220    try:221        req = Requirement(parsed[1])222        if req.name in markers:223            _AmbiguousMarker.emit(field=label, req=parsed[1])224    except InvalidRequirement as ex:225        if any(parsed[1].startswith(marker) for marker in markers):226            msg = _AmbiguousMarker.message(field=label, req=parsed[1])227            raise InvalidRequirement(msg) from ex228 229 230class ConfigHandler(Generic[Target]):231    """Handles metadata supplied in configuration files."""232 233    section_prefix: str234    """Prefix for config sections handled by this handler.235    Must be provided by class heirs.236 237    """238 239    aliases: Dict[str, str] = {}240    """Options aliases.241    For compatibility with various packages. E.g.: d2to1 and pbr.242    Note: `-` in keys is replaced with `_` by config parser.243 244    """245 246    def __init__(247        self,248        target_obj: Target,249        options: AllCommandOptions,250        ignore_option_errors,251        ensure_discovered: expand.EnsurePackagesDiscovered,252    ):253        self.ignore_option_errors = ignore_option_errors254        self.target_obj = target_obj255        self.sections = dict(self._section_options(options))256        self.set_options: List[str] = []257        self.ensure_discovered = ensure_discovered258        self._referenced_files: Set[str] = set()259        """After parsing configurations, this property will enumerate260        all files referenced by the "file:" directive. Private API for setuptools only.261        """262 263    @classmethod264    def _section_options(cls, options: AllCommandOptions):265        for full_name, value in options.items():266            pre, sep, name = full_name.partition(cls.section_prefix)267            if pre:268                continue269            yield name.lstrip('.'), value270 271    @property272    def parsers(self):273        """Metadata item name to parser function mapping."""274        raise NotImplementedError(275            '%s must provide .parsers property' % self.__class__.__name__276        )277 278    def __setitem__(self, option_name, value):279        target_obj = self.target_obj280 281        # Translate alias into real name.282        option_name = self.aliases.get(option_name, option_name)283 284        try:285            current_value = getattr(target_obj, option_name)286        except AttributeError as e:287            raise KeyError(option_name) from e288 289        if current_value:290            # Already inhabited. Skipping.291            return292 293        try:294            parsed = self.parsers.get(option_name, lambda x: x)(value)295        except (Exception,) * self.ignore_option_errors:296            return297 298        simple_setter = functools.partial(target_obj.__setattr__, option_name)299        setter = getattr(target_obj, 'set_%s' % option_name, simple_setter)300        setter(parsed)301 302        self.set_options.append(option_name)303 304    @classmethod305    def _parse_list(cls, value, separator=','):306        """Represents value as a list.307 308        Value is split either by separator (defaults to comma) or by lines.309 310        :param value:311        :param separator: List items separator character.312        :rtype: list313        """314        if isinstance(value, list):  # _get_parser_compound case315            return value316 317        if '\n' in value:318            value = value.splitlines()319        else:320            value = value.split(separator)321 322        return [chunk.strip() for chunk in value if chunk.strip()]323 324    @classmethod325    def _parse_dict(cls, value):326        """Represents value as a dict.327 328        :param value:329        :rtype: dict330        """331        separator = '='332        result = {}333        for line in cls._parse_list(value):334            key, sep, val = line.partition(separator)335            if sep != separator:336                raise OptionError(f"Unable to parse option value to dict: {value}")337            result[key.strip()] = val.strip()338 339        return result340 341    @classmethod342    def _parse_bool(cls, value):343        """Represents value as boolean.344 345        :param value:346        :rtype: bool347        """348        value = value.lower()349        return value in ('1', 'true', 'yes')350 351    @classmethod352    def _exclude_files_parser(cls, key):353        """Returns a parser function to make sure field inputs354        are not files.355 356        Parses a value after getting the key so error messages are357        more informative.358 359        :param key:360        :rtype: callable361        """362 363        def parser(value):364            exclude_directive = 'file:'365            if value.startswith(exclude_directive):366                raise ValueError(367                    'Only strings are accepted for the {0} field, '368                    'files are not accepted'.format(key)369                )370            return value371 372        return parser373 374    def _parse_file(self, value, root_dir: StrPath):375        """Represents value as a string, allowing including text376        from nearest files using `file:` directive.377 378        Directive is sandboxed and won't reach anything outside379        directory with setup.py.380 381        Examples:382            file: README.rst, CHANGELOG.md, src/file.txt383 384        :param str value:385        :rtype: str386        """387        include_directive = 'file:'388 389        if not isinstance(value, str):390            return value391 392        if not value.startswith(include_directive):393            return value394 395        spec = value[len(include_directive) :]396        filepaths = [path.strip() for path in spec.split(',')]397        self._referenced_files.update(filepaths)398        return expand.read_files(filepaths, root_dir)399 400    def _parse_attr(self, value, package_dir, root_dir: StrPath):401        """Represents value as a module attribute.402 403        Examples:404            attr: package.attr405            attr: package.module.attr406 407        :param str value:408        :rtype: str409        """410        attr_directive = 'attr:'411        if not value.startswith(attr_directive):412            return value413 414        attr_desc = value.replace(attr_directive, '')415 416        # Make sure package_dir is populated correctly, so `attr:` directives can work417        package_dir.update(self.ensure_discovered.package_dir)418        return expand.read_attr(attr_desc, package_dir, root_dir)419 420    @classmethod421    def _get_parser_compound(cls, *parse_methods):422        """Returns parser function to represents value as a list.423 424        Parses a value applying given methods one after another.425 426        :param parse_methods:427        :rtype: callable428        """429 430        def parse(value):431            parsed = value432 433            for method in parse_methods:434                parsed = method(parsed)435 436            return parsed437 438        return parse439 440    @classmethod441    def _parse_section_to_dict_with_key(cls, section_options, values_parser):442        """Parses section options into a dictionary.443 444        Applies a given parser to each option in a section.445 446        :param dict section_options:447        :param callable values_parser: function with 2 args corresponding to key, value448        :rtype: dict449        """450        value = {}451        for key, (_, val) in section_options.items():452            value[key] = values_parser(key, val)453        return value454 455    @classmethod456    def _parse_section_to_dict(cls, section_options, values_parser=None):457        """Parses section options into a dictionary.458 459        Optionally applies a given parser to each value.460 461        :param dict section_options:462        :param callable values_parser: function with 1 arg corresponding to option value463        :rtype: dict464        """465        parser = (lambda _, v: values_parser(v)) if values_parser else (lambda _, v: v)466        return cls._parse_section_to_dict_with_key(section_options, parser)467 468    def parse_section(self, section_options):469        """Parses configuration file section.470 471        :param dict section_options:472        """473        for name, (_, value) in section_options.items():474            with contextlib.suppress(KeyError):475                # Keep silent for a new option may appear anytime.476                self[name] = value477 478    def parse(self) -> None:479        """Parses configuration file items from one480        or more related sections.481 482        """483        for section_name, section_options in self.sections.items():484            method_postfix = ''485            if section_name:  # [section.option] variant486                method_postfix = '_%s' % section_name487 488            section_parser_method: Optional[Callable] = getattr(489                self,490                # Dots in section names are translated into dunderscores.491                ('parse_section%s' % method_postfix).replace('.', '__'),492                None,493            )494 495            if section_parser_method is None:496                raise OptionError(497                    "Unsupported distribution option section: "498                    f"[{self.section_prefix}.{section_name}]"499                )500 501            section_parser_method(section_options)502 503    def _deprecated_config_handler(self, func, msg, **kw):504        """this function will wrap around parameters that are deprecated505 506        :param msg: deprecation message507        :param func: function to be wrapped around508        """509 510        @wraps(func)511        def config_handler(*args, **kwargs):512            kw.setdefault("stacklevel", 2)513            _DeprecatedConfig.emit("Deprecated config in `setup.cfg`", msg, **kw)514            return func(*args, **kwargs)515 516        return config_handler517 518 519class ConfigMetadataHandler(ConfigHandler["DistributionMetadata"]):520    section_prefix = 'metadata'521 522    aliases = {523        'home_page': 'url',524        'summary': 'description',525        'classifier': 'classifiers',526        'platform': 'platforms',527    }528 529    strict_mode = False530    """We need to keep it loose, to be partially compatible with531    `pbr` and `d2to1` packages which also uses `metadata` section.532 533    """534 535    def __init__(536        self,537        target_obj: "DistributionMetadata",538        options: AllCommandOptions,539        ignore_option_errors: bool,540        ensure_discovered: expand.EnsurePackagesDiscovered,541        package_dir: Optional[dict] = None,542        root_dir: StrPath = os.curdir,543    ):544        super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)545        self.package_dir = package_dir546        self.root_dir = root_dir547 548    @property549    def parsers(self):550        """Metadata item name to parser function mapping."""551        parse_list = self._parse_list552        parse_file = partial(self._parse_file, root_dir=self.root_dir)553        parse_dict = self._parse_dict554        exclude_files_parser = self._exclude_files_parser555 556        return {557            'platforms': parse_list,558            'keywords': parse_list,559            'provides': parse_list,560            'obsoletes': parse_list,561            'classifiers': self._get_parser_compound(parse_file, parse_list),562            'license': exclude_files_parser('license'),563            'license_files': parse_list,564            'description': parse_file,565            'long_description': parse_file,566            'version': self._parse_version,567            'project_urls': parse_dict,568        }569 570    def _parse_version(self, value):571        """Parses `version` option value.572 573        :param value:574        :rtype: str575 576        """577        version = self._parse_file(value, self.root_dir)578 579        if version != value:580            version = version.strip()581            # Be strict about versions loaded from file because it's easy to582            # accidentally include newlines and other unintended content583            try:584                Version(version)585            except InvalidVersion as e:586                raise OptionError(587                    f'Version loaded from {value} does not '588                    f'comply with PEP 440: {version}'589                ) from e590 591            return version592 593        return expand.version(self._parse_attr(value, self.package_dir, self.root_dir))594 595 596class ConfigOptionsHandler(ConfigHandler["Distribution"]):597    section_prefix = 'options'598 599    def __init__(600        self,601        target_obj: "Distribution",602        options: AllCommandOptions,603        ignore_option_errors: bool,604        ensure_discovered: expand.EnsurePackagesDiscovered,605    ):606        super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)607        self.root_dir = target_obj.src_root608        self.package_dir: Dict[str, str] = {}  # To be filled by `find_packages`609 610    @classmethod611    def _parse_list_semicolon(cls, value):612        return cls._parse_list(value, separator=';')613 614    def _parse_file_in_root(self, value):615        return self._parse_file(value, root_dir=self.root_dir)616 617    def _parse_requirements_list(self, label: str, value: str):618        # Parse a requirements list, either by reading in a `file:`, or a list.619        parsed = self._parse_list_semicolon(self._parse_file_in_root(value))620        _warn_accidental_env_marker_misconfig(label, value, parsed)621        # Filter it to only include lines that are not comments. `parse_list`622        # will have stripped each line and filtered out empties.623        return [line for line in parsed if not line.startswith("#")]624 625    @property626    def parsers(self):627        """Metadata item name to parser function mapping."""628        parse_list = self._parse_list629        parse_bool = self._parse_bool630        parse_dict = self._parse_dict631        parse_cmdclass = self._parse_cmdclass632 633        return {634            'zip_safe': parse_bool,635            'include_package_data': parse_bool,636            'package_dir': parse_dict,637            'scripts': parse_list,638            'eager_resources': parse_list,639            'dependency_links': parse_list,640            'namespace_packages': self._deprecated_config_handler(641                parse_list,642                "The namespace_packages parameter is deprecated, "643                "consider using implicit namespaces instead (PEP 420).",644                # TODO: define due date, see setuptools.dist:check_nsp.645            ),646            'install_requires': partial(647                self._parse_requirements_list, "install_requires"648            ),649            'setup_requires': self._parse_list_semicolon,650            'tests_require': self._parse_list_semicolon,651            'packages': self._parse_packages,652            'entry_points': self._parse_file_in_root,653            'py_modules': parse_list,654            'python_requires': SpecifierSet,655            'cmdclass': parse_cmdclass,656        }657 658    def _parse_cmdclass(self, value):659        package_dir = self.ensure_discovered.package_dir660        return expand.cmdclass(self._parse_dict(value), package_dir, self.root_dir)661 662    def _parse_packages(self, value):663        """Parses `packages` option value.664 665        :param value:666        :rtype: list667        """668        find_directives = ['find:', 'find_namespace:']669        trimmed_value = value.strip()670 671        if trimmed_value not in find_directives:672            return self._parse_list(value)673 674        # Read function arguments from a dedicated section.675        find_kwargs = self.parse_section_packages__find(676            self.sections.get('packages.find', {})677        )678 679        find_kwargs.update(680            namespaces=(trimmed_value == find_directives[1]),681            root_dir=self.root_dir,682            fill_package_dir=self.package_dir,683        )684 685        return expand.find_packages(**find_kwargs)686 687    def parse_section_packages__find(self, section_options):688        """Parses `packages.find` configuration file section.689 690        To be used in conjunction with _parse_packages().691 692        :param dict section_options:693        """694        section_data = self._parse_section_to_dict(section_options, self._parse_list)695 696        valid_keys = ['where', 'include', 'exclude']697 698        find_kwargs = dict([699            (k, v) for k, v in section_data.items() if k in valid_keys and v700        ])701 702        where = find_kwargs.get('where')703        if where is not None:704            find_kwargs['where'] = where[0]  # cast list to single val705 706        return find_kwargs707 708    def parse_section_entry_points(self, section_options):709        """Parses `entry_points` configuration file section.710 711        :param dict section_options:712        """713        parsed = self._parse_section_to_dict(section_options, self._parse_list)714        self['entry_points'] = parsed715 716    def _parse_package_data(self, section_options):717        package_data = self._parse_section_to_dict(section_options, self._parse_list)718        return expand.canonic_package_data(package_data)719 720    def parse_section_package_data(self, section_options):721        """Parses `package_data` configuration file section.722 723        :param dict section_options:724        """725        self['package_data'] = self._parse_package_data(section_options)726 727    def parse_section_exclude_package_data(self, section_options):728        """Parses `exclude_package_data` configuration file section.729 730        :param dict section_options:731        """732        self['exclude_package_data'] = self._parse_package_data(section_options)733 734    def parse_section_extras_require(self, section_options):735        """Parses `extras_require` configuration file section.736 737        :param dict section_options:738        """739        parsed = self._parse_section_to_dict_with_key(740            section_options,741            lambda k, v: self._parse_requirements_list(f"extras_require[{k}]", v),742        )743 744        self['extras_require'] = parsed745 746    def parse_section_data_files(self, section_options):747        """Parses `data_files` configuration file section.748 749        :param dict section_options:750        """751        parsed = self._parse_section_to_dict(section_options, self._parse_list)752        self['data_files'] = expand.canonic_data_files(parsed, self.root_dir)753 754 755class _AmbiguousMarker(SetuptoolsDeprecationWarning):756    _SUMMARY = "Ambiguous requirement marker."757    _DETAILS = """758    One of the parsed requirements in `{field}` looks like a valid environment marker:759 760        {req!r}761 762    Please make sure that the configuration file is correct.763    You can use dangling lines to avoid this problem.764    """765    _SEE_DOCS = "userguide/declarative_config.html#opt-2"766    # TODO: should we include due_date here? Initially introduced in 6 Aug 2022.767    # Does this make sense with latest version of packaging?768 769    @classmethod770    def message(cls, **kw):771        docs = f"https://setuptools.pypa.io/en/latest/{cls._SEE_DOCS}"772        return cls._format(cls._SUMMARY, cls._DETAILS, see_url=docs, format_args=kw)773 774 775class _DeprecatedConfig(SetuptoolsDeprecationWarning):776    _SEE_DOCS = "userguide/declarative_config.html"777