codekingpro/portable-devtools

1import os2import sys3import itertools4from importlib.machinery import EXTENSION_SUFFIXES5from importlib.util import cache_from_source as _compiled_file_name6from typing import Dict, Iterator, List, Tuple7from pathlib import Path8 9from distutils.command.build_ext import build_ext as _du_build_ext10from distutils.ccompiler import new_compiler11from distutils.sysconfig import customize_compiler, get_config_var12from distutils import log13 14from setuptools.errors import BaseError15from setuptools.extension import Extension, Library16 17try:18    # Attempt to use Cython for building extensions, if available19    from Cython.Distutils.build_ext import build_ext as _build_ext  # type: ignore[import-not-found] # Cython not installed on CI tests20 21    # Additionally, assert that the compiler module will load22    # also. Ref #1229.23    __import__('Cython.Compiler.Main')24except ImportError:25    _build_ext = _du_build_ext26 27# make sure _config_vars is initialized28get_config_var("LDSHARED")29# Not publicly exposed in typeshed distutils stubs, but this is done on purpose30# See https://github.com/pypa/setuptools/pull/4228#issuecomment-195985640031from distutils.sysconfig import _config_vars as _CONFIG_VARS  # type: ignore # noqa32 33 34def _customize_compiler_for_shlib(compiler):35    if sys.platform == "darwin":36        # building .dylib requires additional compiler flags on OSX; here we37        # temporarily substitute the pyconfig.h variables so that distutils'38        # 'customize_compiler' uses them before we build the shared libraries.39        tmp = _CONFIG_VARS.copy()40        try:41            # XXX Help!  I don't have any idea whether these are right...42            _CONFIG_VARS['LDSHARED'] = (43                "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup"44            )45            _CONFIG_VARS['CCSHARED'] = " -dynamiclib"46            _CONFIG_VARS['SO'] = ".dylib"47            customize_compiler(compiler)48        finally:49            _CONFIG_VARS.clear()50            _CONFIG_VARS.update(tmp)51    else:52        customize_compiler(compiler)53 54 55have_rtld = False56use_stubs = False57libtype = 'shared'58 59if sys.platform == "darwin":60    use_stubs = True61elif os.name != 'nt':62    try:63        import dl  # type: ignore[import-not-found] # https://github.com/python/mypy/issues/1300264 65        use_stubs = have_rtld = hasattr(dl, 'RTLD_NOW')66    except ImportError:67        pass68 69 70def if_dl(s):71    return s if have_rtld else ''72 73 74def get_abi3_suffix():75    """Return the file extension for an abi3-compliant Extension()"""76    for suffix in EXTENSION_SUFFIXES:77        if '.abi3' in suffix:  # Unix78            return suffix79        elif suffix == '.pyd':  # Windows80            return suffix81    return None82 83 84class build_ext(_build_ext):85    editable_mode: bool = False86    inplace: bool = False87 88    def run(self):89        """Build extensions in build directory, then copy if --inplace"""90        old_inplace, self.inplace = self.inplace, 091        _build_ext.run(self)92        self.inplace = old_inplace93        if old_inplace:94            self.copy_extensions_to_source()95 96    def _get_inplace_equivalent(self, build_py, ext: Extension) -> Tuple[str, str]:97        fullname = self.get_ext_fullname(ext.name)98        filename = self.get_ext_filename(fullname)99        modpath = fullname.split('.')100        package = '.'.join(modpath[:-1])101        package_dir = build_py.get_package_dir(package)102        inplace_file = os.path.join(package_dir, os.path.basename(filename))103        regular_file = os.path.join(self.build_lib, filename)104        return (inplace_file, regular_file)105 106    def copy_extensions_to_source(self):107        build_py = self.get_finalized_command('build_py')108        for ext in self.extensions:109            inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)110 111            # Always copy, even if source is older than destination, to ensure112            # that the right extensions for the current Python/platform are113            # used.114            if os.path.exists(regular_file) or not ext.optional:115                self.copy_file(regular_file, inplace_file, level=self.verbose)116 117            if ext._needs_stub:118                inplace_stub = self._get_equivalent_stub(ext, inplace_file)119                self._write_stub_file(inplace_stub, ext, compile=True)120                # Always compile stub and remove the original (leave the cache behind)121                # (this behaviour was observed in previous iterations of the code)122 123    def _get_equivalent_stub(self, ext: Extension, output_file: str) -> str:124        dir_ = os.path.dirname(output_file)125        _, _, name = ext.name.rpartition(".")126        return f"{os.path.join(dir_, name)}.py"127 128    def _get_output_mapping(self) -> Iterator[Tuple[str, str]]:129        if not self.inplace:130            return131 132        build_py = self.get_finalized_command('build_py')133        opt = self.get_finalized_command('install_lib').optimize or ""134 135        for ext in self.extensions:136            inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)137            yield (regular_file, inplace_file)138 139            if ext._needs_stub:140                # This version of `build_ext` always builds artifacts in another dir,141                # when "inplace=True" is given it just copies them back.142                # This is done in the `copy_extensions_to_source` function, which143                # always compile stub files via `_compile_and_remove_stub`.144                # At the end of the process, a `.pyc` stub file is created without the145                # corresponding `.py`.146 147                inplace_stub = self._get_equivalent_stub(ext, inplace_file)148                regular_stub = self._get_equivalent_stub(ext, regular_file)149                inplace_cache = _compiled_file_name(inplace_stub, optimization=opt)150                output_cache = _compiled_file_name(regular_stub, optimization=opt)151                yield (output_cache, inplace_cache)152 153    def get_ext_filename(self, fullname):154        so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX')155        if so_ext:156            filename = os.path.join(*fullname.split('.')) + so_ext157        else:158            filename = _build_ext.get_ext_filename(self, fullname)159            so_ext = get_config_var('EXT_SUFFIX')160 161        if fullname in self.ext_map:162            ext = self.ext_map[fullname]163            use_abi3 = ext.py_limited_api and get_abi3_suffix()164            if use_abi3:165                filename = filename[: -len(so_ext)]166                so_ext = get_abi3_suffix()167                filename = filename + so_ext168            if isinstance(ext, Library):169                fn, ext = os.path.splitext(filename)170                return self.shlib_compiler.library_filename(fn, libtype)171            elif use_stubs and ext._links_to_dynamic:172                d, fn = os.path.split(filename)173                return os.path.join(d, 'dl-' + fn)174        return filename175 176    def initialize_options(self):177        _build_ext.initialize_options(self)178        self.shlib_compiler = None179        self.shlibs = []180        self.ext_map = {}181        self.editable_mode = False182 183    def finalize_options(self):184        _build_ext.finalize_options(self)185        self.extensions = self.extensions or []186        self.check_extensions_list(self.extensions)187        self.shlibs = [ext for ext in self.extensions if isinstance(ext, Library)]188        if self.shlibs:189            self.setup_shlib_compiler()190        for ext in self.extensions:191            ext._full_name = self.get_ext_fullname(ext.name)192        for ext in self.extensions:193            fullname = ext._full_name194            self.ext_map[fullname] = ext195 196            # distutils 3.1 will also ask for module names197            # XXX what to do with conflicts?198            self.ext_map[fullname.split('.')[-1]] = ext199 200            ltd = self.shlibs and self.links_to_dynamic(ext) or False201            ns = ltd and use_stubs and not isinstance(ext, Library)202            ext._links_to_dynamic = ltd203            ext._needs_stub = ns204            filename = ext._file_name = self.get_ext_filename(fullname)205            libdir = os.path.dirname(os.path.join(self.build_lib, filename))206            if ltd and libdir not in ext.library_dirs:207                ext.library_dirs.append(libdir)208            if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:209                ext.runtime_library_dirs.append(os.curdir)210 211        if self.editable_mode:212            self.inplace = True213 214    def setup_shlib_compiler(self):215        compiler = self.shlib_compiler = new_compiler(216            compiler=self.compiler, dry_run=self.dry_run, force=self.force217        )218        _customize_compiler_for_shlib(compiler)219 220        if self.include_dirs is not None:221            compiler.set_include_dirs(self.include_dirs)222        if self.define is not None:223            # 'define' option is a list of (name,value) tuples224            for name, value in self.define:225                compiler.define_macro(name, value)226        if self.undef is not None:227            for macro in self.undef:228                compiler.undefine_macro(macro)229        if self.libraries is not None:230            compiler.set_libraries(self.libraries)231        if self.library_dirs is not None:232            compiler.set_library_dirs(self.library_dirs)233        if self.rpath is not None:234            compiler.set_runtime_library_dirs(self.rpath)235        if self.link_objects is not None:236            compiler.set_link_objects(self.link_objects)237 238        # hack so distutils' build_extension() builds a library instead239        compiler.link_shared_object = link_shared_object.__get__(compiler)240 241    def get_export_symbols(self, ext):242        if isinstance(ext, Library):243            return ext.export_symbols244        return _build_ext.get_export_symbols(self, ext)245 246    def build_extension(self, ext):247        ext._convert_pyx_sources_to_lang()248        _compiler = self.compiler249        try:250            if isinstance(ext, Library):251                self.compiler = self.shlib_compiler252            _build_ext.build_extension(self, ext)253            if ext._needs_stub:254                build_lib = self.get_finalized_command('build_py').build_lib255                self.write_stub(build_lib, ext)256        finally:257            self.compiler = _compiler258 259    def links_to_dynamic(self, ext):260        """Return true if 'ext' links to a dynamic lib in the same package"""261        # XXX this should check to ensure the lib is actually being built262        # XXX as dynamic, and not just using a locally-found version or a263        # XXX static-compiled version264        libnames = dict.fromkeys([lib._full_name for lib in self.shlibs])265        pkg = '.'.join(ext._full_name.split('.')[:-1] + [''])266        return any(pkg + libname in libnames for libname in ext.libraries)267 268    def get_source_files(self) -> List[str]:269        return [*_build_ext.get_source_files(self), *self._get_internal_depends()]270 271    def _get_internal_depends(self) -> Iterator[str]:272        """Yield ``ext.depends`` that are contained by the project directory"""273        project_root = Path(self.distribution.src_root or os.curdir).resolve()274        depends = (dep for ext in self.extensions for dep in ext.depends)275 276        def skip(orig_path: str, reason: str) -> None:277            log.info(278                "dependency %s won't be automatically "279                "included in the manifest: the path %s",280                orig_path,281                reason,282            )283 284        for dep in depends:285            path = Path(dep)286 287            if path.is_absolute():288                skip(dep, "must be relative")289                continue290 291            if ".." in path.parts:292                skip(dep, "can't have `..` segments")293                continue294 295            try:296                resolved = (project_root / path).resolve(strict=True)297            except OSError:298                skip(dep, "doesn't exist")299                continue300 301            try:302                resolved.relative_to(project_root)303            except ValueError:304                skip(dep, "must be inside the project root")305                continue306 307            yield path.as_posix()308 309    def get_outputs(self) -> List[str]:310        if self.inplace:311            return list(self.get_output_mapping().keys())312        return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs())313 314    def get_output_mapping(self) -> Dict[str, str]:315        """See :class:`setuptools.commands.build.SubCommand`"""316        mapping = self._get_output_mapping()317        return dict(sorted(mapping, key=lambda x: x[0]))318 319    def __get_stubs_outputs(self):320        # assemble the base name for each extension that needs a stub321        ns_ext_bases = (322            os.path.join(self.build_lib, *ext._full_name.split('.'))323            for ext in self.extensions324            if ext._needs_stub325        )326        # pair each base with the extension327        pairs = itertools.product(ns_ext_bases, self.__get_output_extensions())328        return list(base + fnext for base, fnext in pairs)329 330    def __get_output_extensions(self):331        yield '.py'332        yield '.pyc'333        if self.get_finalized_command('build_py').optimize:334            yield '.pyo'335 336    def write_stub(self, output_dir, ext, compile=False):337        stub_file = os.path.join(output_dir, *ext._full_name.split('.')) + '.py'338        self._write_stub_file(stub_file, ext, compile)339 340    def _write_stub_file(self, stub_file: str, ext: Extension, compile=False):341        log.info("writing stub loader for %s to %s", ext._full_name, stub_file)342        if compile and os.path.exists(stub_file):343            raise BaseError(stub_file + " already exists! Please delete.")344        if not self.dry_run:345            f = open(stub_file, 'w')346            f.write(347                '\n'.join([348                    "def __bootstrap__():",349                    "   global __bootstrap__, __file__, __loader__",350                    "   import sys, os, pkg_resources, importlib.util" + if_dl(", dl"),351                    "   __file__ = pkg_resources.resource_filename"352                    "(__name__,%r)" % os.path.basename(ext._file_name),353                    "   del __bootstrap__",354                    "   if '__loader__' in globals():",355                    "       del __loader__",356                    if_dl("   old_flags = sys.getdlopenflags()"),357                    "   old_dir = os.getcwd()",358                    "   try:",359                    "     os.chdir(os.path.dirname(__file__))",360                    if_dl("     sys.setdlopenflags(dl.RTLD_NOW)"),361                    "     spec = importlib.util.spec_from_file_location(",362                    "                __name__, __file__)",363                    "     mod = importlib.util.module_from_spec(spec)",364                    "     spec.loader.exec_module(mod)",365                    "   finally:",366                    if_dl("     sys.setdlopenflags(old_flags)"),367                    "     os.chdir(old_dir)",368                    "__bootstrap__()",369                    "",  # terminal \n370                ])371            )372            f.close()373        if compile:374            self._compile_and_remove_stub(stub_file)375 376    def _compile_and_remove_stub(self, stub_file: str):377        from distutils.util import byte_compile378 379        byte_compile([stub_file], optimize=0, force=True, dry_run=self.dry_run)380        optimize = self.get_finalized_command('install_lib').optimize381        if optimize > 0:382            byte_compile(383                [stub_file],384                optimize=optimize,385                force=True,386                dry_run=self.dry_run,387            )388        if os.path.exists(stub_file) and not self.dry_run:389            os.unlink(stub_file)390 391 392if use_stubs or os.name == 'nt':393    # Build shared libraries394    #395    def link_shared_object(396        self,397        objects,398        output_libname,399        output_dir=None,400        libraries=None,401        library_dirs=None,402        runtime_library_dirs=None,403        export_symbols=None,404        debug=0,405        extra_preargs=None,406        extra_postargs=None,407        build_temp=None,408        target_lang=None,409    ):410        self.link(411            self.SHARED_LIBRARY,412            objects,413            output_libname,414            output_dir,415            libraries,416            library_dirs,417            runtime_library_dirs,418            export_symbols,419            debug,420            extra_preargs,421            extra_postargs,422            build_temp,423            target_lang,424        )425 426else:427    # Build static libraries everywhere else428    libtype = 'static'429 430    def link_shared_object(431        self,432        objects,433        output_libname,434        output_dir=None,435        libraries=None,436        library_dirs=None,437        runtime_library_dirs=None,438        export_symbols=None,439        debug=0,440        extra_preargs=None,441        extra_postargs=None,442        build_temp=None,443        target_lang=None,444    ):445        # XXX we need to either disallow these attrs on Library instances,446        # or warn/abort here if set, or something...447        # libraries=None, library_dirs=None, runtime_library_dirs=None,448        # export_symbols=None, extra_preargs=None, extra_postargs=None,449        # build_temp=None450 451        assert output_dir is None  # distutils build_ext doesn't pass this452        output_dir, filename = os.path.split(output_libname)453        basename, ext = os.path.splitext(filename)454        if self.library_filename("x").startswith('lib'):455            # strip 'lib' prefix; this is kludgy if some platform uses456            # a different prefix457            basename = basename[3:]458 459        self.create_static_lib(objects, basename, output_dir, debug, target_lang)460