codekingpro/portable-devtools

1from __future__ import annotations2 3import fnmatch4import itertools5import operator6import os7import stat8import textwrap9from collections.abc import Iterable, Iterator10from functools import partial11from glob import glob12from pathlib import Path13from typing import Any14 15from more_itertools import unique_everseen16 17from .._path import StrPath, StrPathT18from ..dist import Distribution19from ..warnings import SetuptoolsDeprecationWarning20 21import distutils.command.build_py as orig22import distutils.errors23from distutils.util import convert_path24 25_IMPLICIT_DATA_FILES = ('*.pyi', 'py.typed')26 27 28def make_writable(target) -> None:29    os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)30 31 32class build_py(orig.build_py):33    """Enhanced 'build_py' command that includes data files with packages34 35    The data files are specified via a 'package_data' argument to 'setup()'.36    See 'setuptools.dist.Distribution' for more details.37 38    Also, this version of the 'build_py' command allows you to specify both39    'py_modules' and 'packages' in the same setup operation.40    """41 42    distribution: Distribution  # override distutils.dist.Distribution with setuptools.dist.Distribution43    editable_mode: bool = False44    existing_egg_info_dir: StrPath | None = None  #: Private API, internal use only.45 46    def finalize_options(self) -> None:47        orig.build_py.finalize_options(self)48        self.package_data = self.distribution.package_data49        self.exclude_package_data = self.distribution.exclude_package_data or {}50        if 'data_files' in self.__dict__:51            del self.__dict__['data_files']52 53    def copy_file(  # type: ignore[override] # No overload, no bytes support54        self,55        infile: StrPath,56        outfile: StrPathT,57        preserve_mode: bool = True,58        preserve_times: bool = True,59        link: str | None = None,60        level: object = 1,61    ) -> tuple[StrPathT | str, bool]:62        # Overwrite base class to allow using links63        if link:64            infile = str(Path(infile).resolve())65            outfile = str(Path(outfile).resolve())  # type: ignore[assignment] # Re-assigning a str when outfile is StrPath is ok66        return super().copy_file(  # pyright: ignore[reportReturnType] # pypa/distutils#30967            infile, outfile, preserve_mode, preserve_times, link, level68        )69 70    def run(self) -> None:71        """Build modules, packages, and copy data files to build directory"""72        if not (self.py_modules or self.packages) or self.editable_mode:73            return74 75        if self.py_modules:76            self.build_modules()77 78        if self.packages:79            self.build_packages()80            self.build_package_data()81 82        # Only compile actual .py files, using our base class' idea of what our83        # output files are.84        self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=False))85 86    # Should return "list[tuple[str, str, str, list[str]]] | Any" but can't do without typed distutils on Python 3.12+87    def __getattr__(self, attr: str) -> Any:88        "lazily compute data files"89        if attr == 'data_files':90            self.data_files = self._get_data_files()91            return self.data_files92        return orig.build_py.__getattr__(self, attr)93 94    def _get_data_files(self):95        """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""96        self.analyze_manifest()97        return list(map(self._get_pkg_data_files, self.packages or ()))98 99    def get_data_files_without_manifest(self) -> list[tuple[str, str, str, list[str]]]:100        """101        Generate list of ``(package,src_dir,build_dir,filenames)`` tuples,102        but without triggering any attempt to analyze or build the manifest.103        """104        # Prevent eventual errors from unset `manifest_files`105        # (that would otherwise be set by `analyze_manifest`)106        self.__dict__.setdefault('manifest_files', {})107        return list(map(self._get_pkg_data_files, self.packages or ()))108 109    def _get_pkg_data_files(self, package: str) -> tuple[str, str, str, list[str]]:110        # Locate package source directory111        src_dir = self.get_package_dir(package)112 113        # Compute package build directory114        build_dir = os.path.join(*([self.build_lib] + package.split('.')))115 116        # Strip directory from globbed filenames117        filenames = [118            os.path.relpath(file, src_dir)119            for file in self.find_data_files(package, src_dir)120        ]121        return package, src_dir, build_dir, filenames122 123    def find_data_files(self, package, src_dir):124        """Return filenames for package's data files in 'src_dir'"""125        patterns = self._get_platform_patterns(126            self.package_data,127            package,128            src_dir,129            extra_patterns=_IMPLICIT_DATA_FILES,130        )131        globs_expanded = map(partial(glob, recursive=True), patterns)132        # flatten the expanded globs into an iterable of matches133        globs_matches = itertools.chain.from_iterable(globs_expanded)134        glob_files = filter(os.path.isfile, globs_matches)135        files = itertools.chain(136            self.manifest_files.get(package, []),137            glob_files,138        )139        return self.exclude_data_files(package, src_dir, files)140 141    def get_outputs(self, include_bytecode: bool = True) -> list[str]:  # type: ignore[override] # Using a real boolean instead of 0|1142        """See :class:`setuptools.commands.build.SubCommand`"""143        if self.editable_mode:144            return list(self.get_output_mapping().keys())145        return super().get_outputs(include_bytecode)146 147    def get_output_mapping(self) -> dict[str, str]:148        """See :class:`setuptools.commands.build.SubCommand`"""149        mapping = itertools.chain(150            self._get_package_data_output_mapping(),151            self._get_module_mapping(),152        )153        return dict(sorted(mapping, key=operator.itemgetter(0)))154 155    def _get_module_mapping(self) -> Iterator[tuple[str, str]]:156        """Iterate over all modules producing (dest, src) pairs."""157        for package, module, module_file in self.find_all_modules():158            package = package.split('.')159            filename = self.get_module_outfile(self.build_lib, package, module)160            yield (filename, module_file)161 162    def _get_package_data_output_mapping(self) -> Iterator[tuple[str, str]]:163        """Iterate over package data producing (dest, src) pairs."""164        for package, src_dir, build_dir, filenames in self.data_files:165            for filename in filenames:166                target = os.path.join(build_dir, filename)167                srcfile = os.path.join(src_dir, filename)168                yield (target, srcfile)169 170    def build_package_data(self) -> None:171        """Copy data files into build directory"""172        for target, srcfile in self._get_package_data_output_mapping():173            self.mkpath(os.path.dirname(target))174            _outf, _copied = self.copy_file(srcfile, target)175            make_writable(target)176 177    def analyze_manifest(self) -> None:178        self.manifest_files: dict[str, list[str]] = {}179        if not self.distribution.include_package_data:180            return181        src_dirs: dict[str, str] = {}182        for package in self.packages or ():183            # Locate package source directory184            src_dirs[assert_relative(self.get_package_dir(package))] = package185 186        if (187            self.existing_egg_info_dir188            and Path(self.existing_egg_info_dir, "SOURCES.txt").exists()189        ):190            egg_info_dir = self.existing_egg_info_dir191            manifest = Path(egg_info_dir, "SOURCES.txt")192            files = manifest.read_text(encoding="utf-8").splitlines()193        else:194            self.run_command('egg_info')195            ei_cmd = self.get_finalized_command('egg_info')196            egg_info_dir = ei_cmd.egg_info197            files = ei_cmd.filelist.files198 199        check = _IncludePackageDataAbuse()200        for path in self._filter_build_files(files, egg_info_dir):201            d, f = os.path.split(assert_relative(path))202            prev = None203            oldf = f204            while d and d != prev and d not in src_dirs:205                prev = d206                d, df = os.path.split(d)207                f = os.path.join(df, f)208            if d in src_dirs:209                if f == oldf:210                    if check.is_module(f):211                        continue  # it's a module, not data212                else:213                    importable = check.importable_subpackage(src_dirs[d], f)214                    if importable:215                        check.warn(importable)216                self.manifest_files.setdefault(src_dirs[d], []).append(path)217 218    def _filter_build_files(219        self, files: Iterable[str], egg_info: StrPath220    ) -> Iterator[str]:221        """222        ``build_meta`` may try to create egg_info outside of the project directory,223        and this can be problematic for certain plugins (reported in issue #3500).224 225        Extensions might also include between their sources files created on the226        ``build_lib`` and ``build_temp`` directories.227 228        This function should filter this case of invalid files out.229        """230        build = self.get_finalized_command("build")231        build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base)232        norm_dirs = [os.path.normpath(p) for p in build_dirs if p]233 234        for file in files:235            norm_path = os.path.normpath(file)236            if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs):237                yield file238 239    def get_data_files(self) -> None:240        pass  # Lazily compute data files in _get_data_files() function.241 242    def check_package(self, package, package_dir):243        """Check namespace packages' __init__ for declare_namespace"""244        try:245            return self.packages_checked[package]246        except KeyError:247            pass248 249        init_py = orig.build_py.check_package(self, package, package_dir)250        self.packages_checked[package] = init_py251 252        if not init_py or not self.distribution.namespace_packages:253            return init_py254 255        for pkg in self.distribution.namespace_packages:256            if pkg == package or pkg.startswith(package + '.'):257                break258        else:259            return init_py260 261        with open(init_py, 'rb') as f:262            contents = f.read()263        if b'declare_namespace' not in contents:264            raise distutils.errors.DistutilsError(265                f"Namespace package problem: {package} is a namespace package, but "266                "its\n__init__.py does not call declare_namespace()! Please "267                'fix it.\n(See the setuptools manual under '268                '"Namespace Packages" for details.)\n"'269            )270        return init_py271 272    def initialize_options(self):273        self.packages_checked = {}274        orig.build_py.initialize_options(self)275        self.editable_mode = False276        self.existing_egg_info_dir = None277 278    def get_package_dir(self, package: str) -> str:279        res = orig.build_py.get_package_dir(self, package)280        if self.distribution.src_root is not None:281            return os.path.join(self.distribution.src_root, res)282        return res283 284    def exclude_data_files(self, package, src_dir, files):285        """Filter filenames for package's data files in 'src_dir'"""286        files = list(files)287        patterns = self._get_platform_patterns(288            self.exclude_package_data,289            package,290            src_dir,291        )292        match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)293        # flatten the groups of matches into an iterable of matches294        matches = itertools.chain.from_iterable(match_groups)295        bad = set(matches)296        keepers = (fn for fn in files if fn not in bad)297        # ditch dupes298        return list(unique_everseen(keepers))299 300    @staticmethod301    def _get_platform_patterns(spec, package, src_dir, extra_patterns=()):302        """303        yield platform-specific path patterns (suitable for glob304        or fn_match) from a glob-based spec (such as305        self.package_data or self.exclude_package_data)306        matching package in src_dir.307        """308        raw_patterns = itertools.chain(309            extra_patterns,310            spec.get('', []),311            spec.get(package, []),312        )313        return (314            # Each pattern has to be converted to a platform-specific path315            os.path.join(src_dir, convert_path(pattern))316            for pattern in raw_patterns317        )318 319 320def assert_relative(path):321    if not os.path.isabs(path):322        return path323    from distutils.errors import DistutilsSetupError324 325    msg = (326        textwrap.dedent(327            """328        Error: setup script specifies an absolute path:329 330            %s331 332        setup() arguments must *always* be /-separated paths relative to the333        setup.py directory, *never* absolute paths.334        """335        ).lstrip()336        % path337    )338    raise DistutilsSetupError(msg)339 340 341class _IncludePackageDataAbuse:342    """Inform users that package or module is included as 'data file'"""343 344    class _Warning(SetuptoolsDeprecationWarning):345        _SUMMARY = """346        Package {importable!r} is absent from the `packages` configuration.347        """348 349        _DETAILS = """350        ############################351        # Package would be ignored #352        ############################353        Python recognizes {importable!r} as an importable package[^1],354        but it is absent from setuptools' `packages` configuration.355 356        This leads to an ambiguous overall configuration. If you want to distribute this357        package, please make sure that {importable!r} is explicitly added358        to the `packages` configuration field.359 360        Alternatively, you can also rely on setuptools' discovery methods361        (for example by using `find_namespace_packages(...)`/`find_namespace:`362        instead of `find_packages(...)`/`find:`).363 364        You can read more about "package discovery" on setuptools documentation page:365 366        - https://setuptools.pypa.io/en/latest/userguide/package_discovery.html367 368        If you don't want {importable!r} to be distributed and are369        already explicitly excluding {importable!r} via370        `find_namespace_packages(...)/find_namespace` or `find_packages(...)/find`,371        you can try to use `exclude_package_data`, or `include-package-data=False` in372        combination with a more fine grained `package-data` configuration.373 374        You can read more about "package data files" on setuptools documentation page:375 376        - https://setuptools.pypa.io/en/latest/userguide/datafiles.html377 378 379        [^1]: For Python, any directory (with suitable naming) can be imported,380              even if it does not contain any `.py` files.381              On the other hand, currently there is no concept of package data382              directory, all directories are treated like packages.383        """384        # _DUE_DATE: still not defined as this is particularly controversial.385        # Warning initially introduced in May 2022. See issue #3340 for discussion.386 387    def __init__(self) -> None:388        self._already_warned = set[str]()389 390    def is_module(self, file):391        return file.endswith(".py") and file[: -len(".py")].isidentifier()392 393    def importable_subpackage(self, parent, file):394        pkg = Path(file).parent395        parts = list(itertools.takewhile(str.isidentifier, pkg.parts))396        if parts:397            return ".".join([parent, *parts])398        return None399 400    def warn(self, importable):401        if importable not in self._already_warned:402            self._Warning.emit(importable=importable)403            self._already_warned.add(importable)404