codekingpro/portable-devtools

1from functools import partial2from glob import glob3from distutils.util import convert_path4import distutils.command.build_py as orig5import os6import fnmatch7import textwrap8import distutils.errors9import itertools10import stat11from pathlib import Path12from typing import Dict, Iterable, Iterator, List, Optional, Tuple13 14from ..extern.more_itertools import unique_everseen15from ..warnings import SetuptoolsDeprecationWarning16 17 18_IMPLICIT_DATA_FILES = ('*.pyi', 'py.typed')19 20 21def make_writable(target):22    os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)23 24 25class build_py(orig.build_py):26    """Enhanced 'build_py' command that includes data files with packages27 28    The data files are specified via a 'package_data' argument to 'setup()'.29    See 'setuptools.dist.Distribution' for more details.30 31    Also, this version of the 'build_py' command allows you to specify both32    'py_modules' and 'packages' in the same setup operation.33    """34 35    editable_mode: bool = False36    existing_egg_info_dir: Optional[str] = None  #: Private API, internal use only.37 38    def finalize_options(self):39        orig.build_py.finalize_options(self)40        self.package_data = self.distribution.package_data41        self.exclude_package_data = self.distribution.exclude_package_data or {}42        if 'data_files' in self.__dict__:43            del self.__dict__['data_files']44        self.__updated_files = []45 46    def copy_file(47        self, infile, outfile, preserve_mode=1, preserve_times=1, link=None, level=148    ):49        # Overwrite base class to allow using links50        if link:51            infile = str(Path(infile).resolve())52            outfile = str(Path(outfile).resolve())53        return super().copy_file(54            infile, outfile, preserve_mode, preserve_times, link, level55        )56 57    def run(self):58        """Build modules, packages, and copy data files to build directory"""59        if not (self.py_modules or self.packages) or self.editable_mode:60            return61 62        if self.py_modules:63            self.build_modules()64 65        if self.packages:66            self.build_packages()67            self.build_package_data()68 69        # Only compile actual .py files, using our base class' idea of what our70        # output files are.71        self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))72 73    def __getattr__(self, attr):74        "lazily compute data files"75        if attr == 'data_files':76            self.data_files = self._get_data_files()77            return self.data_files78        return orig.build_py.__getattr__(self, attr)79 80    def build_module(self, module, module_file, package):81        outfile, copied = orig.build_py.build_module(self, module, module_file, package)82        if copied:83            self.__updated_files.append(outfile)84        return outfile, copied85 86    def _get_data_files(self):87        """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""88        self.analyze_manifest()89        return list(map(self._get_pkg_data_files, self.packages or ()))90 91    def get_data_files_without_manifest(self):92        """93        Generate list of ``(package,src_dir,build_dir,filenames)`` tuples,94        but without triggering any attempt to analyze or build the manifest.95        """96        # Prevent eventual errors from unset `manifest_files`97        # (that would otherwise be set by `analyze_manifest`)98        self.__dict__.setdefault('manifest_files', {})99        return list(map(self._get_pkg_data_files, self.packages or ()))100 101    def _get_pkg_data_files(self, package):102        # Locate package source directory103        src_dir = self.get_package_dir(package)104 105        # Compute package build directory106        build_dir = os.path.join(*([self.build_lib] + package.split('.')))107 108        # Strip directory from globbed filenames109        filenames = [110            os.path.relpath(file, src_dir)111            for file in self.find_data_files(package, src_dir)112        ]113        return package, src_dir, build_dir, filenames114 115    def find_data_files(self, package, src_dir):116        """Return filenames for package's data files in 'src_dir'"""117        patterns = self._get_platform_patterns(118            self.package_data,119            package,120            src_dir,121            extra_patterns=_IMPLICIT_DATA_FILES,122        )123        globs_expanded = map(partial(glob, recursive=True), patterns)124        # flatten the expanded globs into an iterable of matches125        globs_matches = itertools.chain.from_iterable(globs_expanded)126        glob_files = filter(os.path.isfile, globs_matches)127        files = itertools.chain(128            self.manifest_files.get(package, []),129            glob_files,130        )131        return self.exclude_data_files(package, src_dir, files)132 133    def get_outputs(self, include_bytecode=1) -> List[str]:134        """See :class:`setuptools.commands.build.SubCommand`"""135        if self.editable_mode:136            return list(self.get_output_mapping().keys())137        return super().get_outputs(include_bytecode)138 139    def get_output_mapping(self) -> Dict[str, str]:140        """See :class:`setuptools.commands.build.SubCommand`"""141        mapping = itertools.chain(142            self._get_package_data_output_mapping(),143            self._get_module_mapping(),144        )145        return dict(sorted(mapping, key=lambda x: x[0]))146 147    def _get_module_mapping(self) -> Iterator[Tuple[str, str]]:148        """Iterate over all modules producing (dest, src) pairs."""149        for package, module, module_file in self.find_all_modules():150            package = package.split('.')151            filename = self.get_module_outfile(self.build_lib, package, module)152            yield (filename, module_file)153 154    def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]:155        """Iterate over package data producing (dest, src) pairs."""156        for package, src_dir, build_dir, filenames in self.data_files:157            for filename in filenames:158                target = os.path.join(build_dir, filename)159                srcfile = os.path.join(src_dir, filename)160                yield (target, srcfile)161 162    def build_package_data(self):163        """Copy data files into build directory"""164        for target, srcfile in self._get_package_data_output_mapping():165            self.mkpath(os.path.dirname(target))166            _outf, _copied = self.copy_file(srcfile, target)167            make_writable(target)168 169    def analyze_manifest(self):170        self.manifest_files = mf = {}171        if not self.distribution.include_package_data:172            return173        src_dirs = {}174        for package in self.packages or ():175            # Locate package source directory176            src_dirs[assert_relative(self.get_package_dir(package))] = package177 178        if (179            getattr(self, 'existing_egg_info_dir', None)180            and Path(self.existing_egg_info_dir, "SOURCES.txt").exists()181        ):182            egg_info_dir = self.existing_egg_info_dir183            manifest = Path(egg_info_dir, "SOURCES.txt")184            files = manifest.read_text(encoding="utf-8").splitlines()185        else:186            self.run_command('egg_info')187            ei_cmd = self.get_finalized_command('egg_info')188            egg_info_dir = ei_cmd.egg_info189            files = ei_cmd.filelist.files190 191        check = _IncludePackageDataAbuse()192        for path in self._filter_build_files(files, egg_info_dir):193            d, f = os.path.split(assert_relative(path))194            prev = None195            oldf = f196            while d and d != prev and d not in src_dirs:197                prev = d198                d, df = os.path.split(d)199                f = os.path.join(df, f)200            if d in src_dirs:201                if f == oldf:202                    if check.is_module(f):203                        continue  # it's a module, not data204                else:205                    importable = check.importable_subpackage(src_dirs[d], f)206                    if importable:207                        check.warn(importable)208                mf.setdefault(src_dirs[d], []).append(path)209 210    def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]:211        """212        ``build_meta`` may try to create egg_info outside of the project directory,213        and this can be problematic for certain plugins (reported in issue #3500).214 215        Extensions might also include between their sources files created on the216        ``build_lib`` and ``build_temp`` directories.217 218        This function should filter this case of invalid files out.219        """220        build = self.get_finalized_command("build")221        build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base)222        norm_dirs = [os.path.normpath(p) for p in build_dirs if p]223 224        for file in files:225            norm_path = os.path.normpath(file)226            if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs):227                yield file228 229    def get_data_files(self):230        pass  # Lazily compute data files in _get_data_files() function.231 232    def check_package(self, package, package_dir):233        """Check namespace packages' __init__ for declare_namespace"""234        try:235            return self.packages_checked[package]236        except KeyError:237            pass238 239        init_py = orig.build_py.check_package(self, package, package_dir)240        self.packages_checked[package] = init_py241 242        if not init_py or not self.distribution.namespace_packages:243            return init_py244 245        for pkg in self.distribution.namespace_packages:246            if pkg == package or pkg.startswith(package + '.'):247                break248        else:249            return init_py250 251        with open(init_py, 'rb') as f:252            contents = f.read()253        if b'declare_namespace' not in contents:254            raise distutils.errors.DistutilsError(255                "Namespace package problem: %s is a namespace package, but "256                "its\n__init__.py does not call declare_namespace()! Please "257                'fix it.\n(See the setuptools manual under '258                '"Namespace Packages" for details.)\n"' % (package,)259            )260        return init_py261 262    def initialize_options(self):263        self.packages_checked = {}264        orig.build_py.initialize_options(self)265        self.editable_mode = False266        self.existing_egg_info_dir = None267 268    def get_package_dir(self, package):269        res = orig.build_py.get_package_dir(self, package)270        if self.distribution.src_root is not None:271            return os.path.join(self.distribution.src_root, res)272        return res273 274    def exclude_data_files(self, package, src_dir, files):275        """Filter filenames for package's data files in 'src_dir'"""276        files = list(files)277        patterns = self._get_platform_patterns(278            self.exclude_package_data,279            package,280            src_dir,281        )282        match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)283        # flatten the groups of matches into an iterable of matches284        matches = itertools.chain.from_iterable(match_groups)285        bad = set(matches)286        keepers = (fn for fn in files if fn not in bad)287        # ditch dupes288        return list(unique_everseen(keepers))289 290    @staticmethod291    def _get_platform_patterns(spec, package, src_dir, extra_patterns=()):292        """293        yield platform-specific path patterns (suitable for glob294        or fn_match) from a glob-based spec (such as295        self.package_data or self.exclude_package_data)296        matching package in src_dir.297        """298        raw_patterns = itertools.chain(299            extra_patterns,300            spec.get('', []),301            spec.get(package, []),302        )303        return (304            # Each pattern has to be converted to a platform-specific path305            os.path.join(src_dir, convert_path(pattern))306            for pattern in raw_patterns307        )308 309 310def assert_relative(path):311    if not os.path.isabs(path):312        return path313    from distutils.errors import DistutilsSetupError314 315    msg = (316        textwrap.dedent(317            """318        Error: setup script specifies an absolute path:319 320            %s321 322        setup() arguments must *always* be /-separated paths relative to the323        setup.py directory, *never* absolute paths.324        """325        ).lstrip()326        % path327    )328    raise DistutilsSetupError(msg)329 330 331class _IncludePackageDataAbuse:332    """Inform users that package or module is included as 'data file'"""333 334    class _Warning(SetuptoolsDeprecationWarning):335        _SUMMARY = """336        Package {importable!r} is absent from the `packages` configuration.337        """338 339        _DETAILS = """340        ############################341        # Package would be ignored #342        ############################343        Python recognizes {importable!r} as an importable package[^1],344        but it is absent from setuptools' `packages` configuration.345 346        This leads to an ambiguous overall configuration. If you want to distribute this347        package, please make sure that {importable!r} is explicitly added348        to the `packages` configuration field.349 350        Alternatively, you can also rely on setuptools' discovery methods351        (for example by using `find_namespace_packages(...)`/`find_namespace:`352        instead of `find_packages(...)`/`find:`).353 354        You can read more about "package discovery" on setuptools documentation page:355 356        - https://setuptools.pypa.io/en/latest/userguide/package_discovery.html357 358        If you don't want {importable!r} to be distributed and are359        already explicitly excluding {importable!r} via360        `find_namespace_packages(...)/find_namespace` or `find_packages(...)/find`,361        you can try to use `exclude_package_data`, or `include-package-data=False` in362        combination with a more fine grained `package-data` configuration.363 364        You can read more about "package data files" on setuptools documentation page:365 366        - https://setuptools.pypa.io/en/latest/userguide/datafiles.html367 368 369        [^1]: For Python, any directory (with suitable naming) can be imported,370              even if it does not contain any `.py` files.371              On the other hand, currently there is no concept of package data372              directory, all directories are treated like packages.373        """374        # _DUE_DATE: still not defined as this is particularly controversial.375        # Warning initially introduced in May 2022. See issue #3340 for discussion.376 377    def __init__(self):378        self._already_warned = set()379 380    def is_module(self, file):381        return file.endswith(".py") and file[: -len(".py")].isidentifier()382 383    def importable_subpackage(self, parent, file):384        pkg = Path(file).parent385        parts = list(itertools.takewhile(str.isidentifier, pkg.parts))386        if parts:387            return ".".join([parent, *parts])388        return None389 390    def warn(self, importable):391        if importable not in self._already_warned:392            self._Warning.emit(importable=importable)393            self._already_warned.add(importable)394