codekingpro/portable-devtools

1from __future__ import annotations2 3import itertools4import operator5import os6import sys7import textwrap8from collections.abc import Iterator9from importlib.machinery import EXTENSION_SUFFIXES10from importlib.util import cache_from_source as _compiled_file_name11from pathlib import Path12from typing import TYPE_CHECKING13 14from setuptools.dist import Distribution15from setuptools.errors import BaseError16from setuptools.extension import Extension, Library17 18from distutils import log19from distutils.ccompiler import new_compiler20from distutils.sysconfig import customize_compiler, get_config_var21 22if TYPE_CHECKING:23    # Cython not installed on CI tests, causing _build_ext to be `Any`24    from distutils.command.build_ext import build_ext as _build_ext25else:26    try:27        # Attempt to use Cython for building extensions, if available28        from Cython.Distutils.build_ext import build_ext as _build_ext29 30        # Additionally, assert that the compiler module will load31        # also. Ref #1229.32        __import__('Cython.Compiler.Main')33    except ImportError:34        from distutils.command.build_ext import build_ext as _build_ext35 36# make sure _config_vars is initialized37get_config_var("LDSHARED")38# Not publicly exposed in typeshed distutils stubs, but this is done on purpose39# See https://github.com/pypa/setuptools/pull/4228#issuecomment-195985640040from distutils.sysconfig import _config_vars as _CONFIG_VARS  # noqa: E40241 42 43def _customize_compiler_for_shlib(compiler):44    if sys.platform == "darwin":45        # building .dylib requires additional compiler flags on OSX; here we46        # temporarily substitute the pyconfig.h variables so that distutils'47        # 'customize_compiler' uses them before we build the shared libraries.48        tmp = _CONFIG_VARS.copy()49        try:50            # XXX Help!  I don't have any idea whether these are right...51            _CONFIG_VARS['LDSHARED'] = (52                "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup"53            )54            _CONFIG_VARS['CCSHARED'] = " -dynamiclib"55            _CONFIG_VARS['SO'] = ".dylib"56            customize_compiler(compiler)57        finally:58            _CONFIG_VARS.clear()59            _CONFIG_VARS.update(tmp)60    else:61        customize_compiler(compiler)62 63 64have_rtld = False65use_stubs = False66libtype = 'shared'67 68if sys.platform == "darwin":69    use_stubs = True70elif os.name != 'nt':71    try:72        import dl  # type: ignore[import-not-found] # https://github.com/python/mypy/issues/1300273 74        use_stubs = have_rtld = hasattr(dl, 'RTLD_NOW')75    except ImportError:76        pass77 78 79def get_abi3_suffix():80    """Return the file extension for an abi3-compliant Extension()"""81    for suffix in EXTENSION_SUFFIXES:82        if '.abi3' in suffix:  # Unix83            return suffix84        elif suffix == '.pyd':  # Windows85            return suffix86    return None87 88 89class build_ext(_build_ext):90    distribution: Distribution  # override distutils.dist.Distribution with setuptools.dist.Distribution91    editable_mode = False92    inplace = False93 94    def run(self) -> None:95        """Build extensions in build directory, then copy if --inplace"""96        old_inplace, self.inplace = self.inplace, False97        _build_ext.run(self)98        self.inplace = old_inplace99        if old_inplace:100            self.copy_extensions_to_source()101 102    def _get_inplace_equivalent(self, build_py, ext: Extension) -> tuple[str, str]:103        fullname = self.get_ext_fullname(ext.name)104        filename = self.get_ext_filename(fullname)105        modpath = fullname.split('.')106        package = '.'.join(modpath[:-1])107        package_dir = build_py.get_package_dir(package)108        inplace_file = os.path.join(package_dir, os.path.basename(filename))109        regular_file = os.path.join(self.build_lib, filename)110        return (inplace_file, regular_file)111 112    def copy_extensions_to_source(self) -> None:113        build_py = self.get_finalized_command('build_py')114        for ext in self.extensions:115            inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)116 117            # Always copy, even if source is older than destination, to ensure118            # that the right extensions for the current Python/platform are119            # used.120            if os.path.exists(regular_file) or not ext.optional:121                self.copy_file(regular_file, inplace_file, level=self.verbose)122 123            if ext._needs_stub:124                inplace_stub = self._get_equivalent_stub(ext, inplace_file)125                self._write_stub_file(inplace_stub, ext, compile=True)126                # Always compile stub and remove the original (leave the cache behind)127                # (this behaviour was observed in previous iterations of the code)128 129    def _get_equivalent_stub(self, ext: Extension, output_file: str) -> str:130        dir_ = os.path.dirname(output_file)131        _, _, name = ext.name.rpartition(".")132        return f"{os.path.join(dir_, name)}.py"133 134    def _get_output_mapping(self) -> Iterator[tuple[str, str]]:135        if not self.inplace:136            return137 138        build_py = self.get_finalized_command('build_py')139        opt = self.get_finalized_command('install_lib').optimize or ""140 141        for ext in self.extensions:142            inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)143            yield (regular_file, inplace_file)144 145            if ext._needs_stub:146                # This version of `build_ext` always builds artifacts in another dir,147                # when "inplace=True" is given it just copies them back.148                # This is done in the `copy_extensions_to_source` function, which149                # always compile stub files via `_compile_and_remove_stub`.150                # At the end of the process, a `.pyc` stub file is created without the151                # corresponding `.py`.152 153                inplace_stub = self._get_equivalent_stub(ext, inplace_file)154                regular_stub = self._get_equivalent_stub(ext, regular_file)155                inplace_cache = _compiled_file_name(inplace_stub, optimization=opt)156                output_cache = _compiled_file_name(regular_stub, optimization=opt)157                yield (output_cache, inplace_cache)158 159    def get_ext_filename(self, fullname: str) -> str:160        so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX')161        if so_ext:162            filename = os.path.join(*fullname.split('.')) + so_ext163        else:164            filename = _build_ext.get_ext_filename(self, fullname)165            ext_suffix = get_config_var('EXT_SUFFIX')166            if not isinstance(ext_suffix, str):167                raise OSError(168                    "Configuration variable EXT_SUFFIX not found for this platform "169                    "and environment variable SETUPTOOLS_EXT_SUFFIX is missing"170                )171            so_ext = ext_suffix172 173        if fullname in self.ext_map:174            ext = self.ext_map[fullname]175            abi3_suffix = get_abi3_suffix()176            if ext.py_limited_api and abi3_suffix:  # Use abi3177                filename = filename[: -len(so_ext)] + abi3_suffix178            if isinstance(ext, Library):179                fn, ext = os.path.splitext(filename)180                return self.shlib_compiler.library_filename(fn, libtype)181            elif use_stubs and ext._links_to_dynamic:182                d, fn = os.path.split(filename)183                return os.path.join(d, 'dl-' + fn)184        return filename185 186    def initialize_options(self):187        _build_ext.initialize_options(self)188        self.shlib_compiler = None189        self.shlibs = []190        self.ext_map = {}191        self.editable_mode = False192 193    def finalize_options(self) -> None:194        _build_ext.finalize_options(self)195        self.extensions = self.extensions or []196        self.check_extensions_list(self.extensions)197        self.shlibs = [ext for ext in self.extensions if isinstance(ext, Library)]198        if self.shlibs:199            self.setup_shlib_compiler()200        for ext in self.extensions:201            ext._full_name = self.get_ext_fullname(ext.name)202        for ext in self.extensions:203            fullname = ext._full_name204            self.ext_map[fullname] = ext205 206            # distutils 3.1 will also ask for module names207            # XXX what to do with conflicts?208            self.ext_map[fullname.split('.')[-1]] = ext209 210            ltd = self.shlibs and self.links_to_dynamic(ext) or False211            ns = ltd and use_stubs and not isinstance(ext, Library)212            ext._links_to_dynamic = ltd213            ext._needs_stub = ns214            filename = ext._file_name = self.get_ext_filename(fullname)215            libdir = os.path.dirname(os.path.join(self.build_lib, filename))216            if ltd and libdir not in ext.library_dirs:217                ext.library_dirs.append(libdir)218            if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:219                ext.runtime_library_dirs.append(os.curdir)220 221        if self.editable_mode:222            self.inplace = True223 224    def setup_shlib_compiler(self) -> None:225        compiler = self.shlib_compiler = new_compiler(226            compiler=self.compiler, force=self.force227        )228        _customize_compiler_for_shlib(compiler)229 230        if self.include_dirs is not None:231            compiler.set_include_dirs(self.include_dirs)232        if self.define is not None:233            # 'define' option is a list of (name,value) tuples234            for name, value in self.define:235                compiler.define_macro(name, value)236        if self.undef is not None:237            for macro in self.undef:238                compiler.undefine_macro(macro)239        if self.libraries is not None:240            compiler.set_libraries(self.libraries)241        if self.library_dirs is not None:242            compiler.set_library_dirs(self.library_dirs)243        if self.rpath is not None:244            compiler.set_runtime_library_dirs(self.rpath)245        if self.link_objects is not None:246            compiler.set_link_objects(self.link_objects)247 248        # hack so distutils' build_extension() builds a library instead249        compiler.link_shared_object = link_shared_object.__get__(compiler)  # type: ignore[method-assign]250 251    def get_export_symbols(self, ext):252        if isinstance(ext, Library):253            return ext.export_symbols254        return _build_ext.get_export_symbols(self, ext)255 256    def build_extension(self, ext) -> None:257        ext._convert_pyx_sources_to_lang()258        _compiler = self.compiler259        try:260            if isinstance(ext, Library):261                self.compiler = self.shlib_compiler262            _build_ext.build_extension(self, ext)263            if ext._needs_stub:264                build_lib = self.get_finalized_command('build_py').build_lib265                self.write_stub(build_lib, ext)266        finally:267            self.compiler = _compiler268 269    def links_to_dynamic(self, ext):270        """Return true if 'ext' links to a dynamic lib in the same package"""271        # XXX this should check to ensure the lib is actually being built272        # XXX as dynamic, and not just using a locally-found version or a273        # XXX static-compiled version274        libnames = dict.fromkeys([lib._full_name for lib in self.shlibs])275        pkg = '.'.join(ext._full_name.split('.')[:-1] + [''])276        return any(pkg + libname in libnames for libname in ext.libraries)277 278    def get_source_files(self) -> list[str]:279        return [*_build_ext.get_source_files(self), *self._get_internal_depends()]280 281    def _get_internal_depends(self) -> Iterator[str]:282        """Yield ``ext.depends`` that are contained by the project directory"""283        project_root = Path(self.distribution.src_root or os.curdir).resolve()284        depends = (dep for ext in self.extensions for dep in ext.depends)285 286        def skip(orig_path: str, reason: str) -> None:287            log.info(288                "dependency %s won't be automatically "289                "included in the manifest: the path %s",290                orig_path,291                reason,292            )293 294        for dep in depends:295            path = Path(dep)296 297            if path.is_absolute():298                skip(dep, "must be relative")299                continue300 301            if ".." in path.parts:302                skip(dep, "can't have `..` segments")303                continue304 305            try:306                resolved = (project_root / path).resolve(strict=True)307            except OSError:308                skip(dep, "doesn't exist")309                continue310 311            try:312                resolved.relative_to(project_root)313            except ValueError:314                skip(dep, "must be inside the project root")315                continue316 317            yield path.as_posix()318 319    def get_outputs(self) -> list[str]:320        if self.inplace:321            return list(self.get_output_mapping().keys())322        return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs())323 324    def get_output_mapping(self) -> dict[str, str]:325        """See :class:`setuptools.commands.build.SubCommand`"""326        mapping = self._get_output_mapping()327        return dict(sorted(mapping, key=operator.itemgetter(0)))328 329    def __get_stubs_outputs(self):330        # assemble the base name for each extension that needs a stub331        ns_ext_bases = (332            os.path.join(self.build_lib, *ext._full_name.split('.'))333            for ext in self.extensions334            if ext._needs_stub335        )336        # pair each base with the extension337        pairs = itertools.product(ns_ext_bases, self.__get_output_extensions())338        return list(base + fnext for base, fnext in pairs)339 340    def __get_output_extensions(self):341        yield '.py'342        yield '.pyc'343        if self.get_finalized_command('build_py').optimize:344            yield '.pyo'345 346    def write_stub(self, output_dir, ext, compile=False) -> None:347        stub_file = os.path.join(output_dir, *ext._full_name.split('.')) + '.py'348        self._write_stub_file(stub_file, ext, compile)349 350    def _write_stub_file(self, stub_file: str, ext: Extension, compile=False):351        log.info("writing stub loader for %s to %s", ext._full_name, stub_file)352        if compile and os.path.exists(stub_file):353            raise BaseError(stub_file + " already exists! Please delete.")354        with open(stub_file, 'w', encoding="utf-8") as f:355            content = (356                textwrap357                .dedent(f"""358                def __bootstrap__():359                   global __bootstrap__, __file__, __loader__360                   import sys, os, importlib.resources as irs, importlib.util361                #rtld   import dl362                   with irs.files(__name__).joinpath(363                     {os.path.basename(ext._file_name)!r}) as __file__:364                      del __bootstrap__365                      if '__loader__' in globals():366                          del __loader__367                #rtld      old_flags = sys.getdlopenflags()368                      old_dir = os.getcwd()369                      try:370                        os.chdir(os.path.dirname(__file__))371                #rtld        sys.setdlopenflags(dl.RTLD_NOW)372                        spec = importlib.util.spec_from_file_location(373                                   __name__, __file__)374                        mod = importlib.util.module_from_spec(spec)375                        spec.loader.exec_module(mod)376                      finally:377                #rtld        sys.setdlopenflags(old_flags)378                        os.chdir(old_dir)379                __bootstrap__()380                """)381                .lstrip()382                .replace('#rtld', '#rtld' * (not have_rtld))383            )384            f.write(content)385        if compile:386            self._compile_and_remove_stub(stub_file)387 388    def _compile_and_remove_stub(self, stub_file: str):389        from distutils.util import byte_compile390 391        byte_compile([stub_file], optimize=0, force=True)392        optimize = self.get_finalized_command('install_lib').optimize393        if optimize > 0:394            byte_compile(395                [stub_file],396                optimize=optimize,397                force=True,398            )399        if os.path.exists(stub_file):400            os.unlink(stub_file)401 402 403if use_stubs or os.name == 'nt':404    # Build shared libraries405    #406    def link_shared_object(407        self,408        objects,409        output_libname,410        output_dir=None,411        libraries=None,412        library_dirs=None,413        runtime_library_dirs=None,414        export_symbols=None,415        debug: bool = False,416        extra_preargs=None,417        extra_postargs=None,418        build_temp=None,419        target_lang=None,420    ) -> None:421        self.link(422            self.SHARED_LIBRARY,423            objects,424            output_libname,425            output_dir,426            libraries,427            library_dirs,428            runtime_library_dirs,429            export_symbols,430            debug,431            extra_preargs,432            extra_postargs,433            build_temp,434            target_lang,435        )436 437else:438    # Build static libraries everywhere else439    libtype = 'static'440 441    def link_shared_object(442        self,443        objects,444        output_libname,445        output_dir=None,446        libraries=None,447        library_dirs=None,448        runtime_library_dirs=None,449        export_symbols=None,450        debug: bool = False,451        extra_preargs=None,452        extra_postargs=None,453        build_temp=None,454        target_lang=None,455    ) -> None:456        # XXX we need to either disallow these attrs on Library instances,457        # or warn/abort here if set, or something...458        # libraries=None, library_dirs=None, runtime_library_dirs=None,459        # export_symbols=None, extra_preargs=None, extra_postargs=None,460        # build_temp=None461 462        assert output_dir is None  # distutils build_ext doesn't pass this463        output_dir, filename = os.path.split(output_libname)464        basename, _ext = os.path.splitext(filename)465        if self.library_filename("x").startswith('lib'):466            # strip 'lib' prefix; this is kludgy if some platform uses467            # a different prefix468            basename = basename[3:]469 470        self.create_static_lib(objects, basename, output_dir, debug, target_lang)471