codekingpro/portable-devtools

1import contextlib2import glob3import importlib4import os.path5import platform6import re7import shutil8import site9import subprocess10import sys11import tempfile12import textwrap13import time14from distutils import sysconfig15from distutils.command.build_ext import build_ext16from distutils.core import Distribution17from distutils.errors import (18    CompileError,19    DistutilsPlatformError,20    DistutilsSetupError,21    UnknownFileError,22)23from distutils.extension import Extension24from distutils.tests import missing_compiler_executable25from distutils.tests.support import TempdirManager, copy_xxmodule_c, fixup_build_ext26from io import StringIO27 28import jaraco.path29import path30import pytest31from test import support32 33from .compat import py39 as import_helper34 35 36@pytest.fixture()37def user_site_dir(request):38    self = request.instance39    self.tmp_dir = self.mkdtemp()40    self.tmp_path = path.Path(self.tmp_dir)41    from distutils.command import build_ext42 43    orig_user_base = site.USER_BASE44 45    site.USER_BASE = self.mkdtemp()46    build_ext.USER_BASE = site.USER_BASE47 48    # bpo-30132: On Windows, a .pdb file may be created in the current49    # working directory. Create a temporary working directory to cleanup50    # everything at the end of the test.51    with self.tmp_path:52        yield53 54    site.USER_BASE = orig_user_base55    build_ext.USER_BASE = orig_user_base56 57    if sys.platform == 'cygwin':58        time.sleep(1)59 60 61@contextlib.contextmanager62def safe_extension_import(name, path):63    with import_helper.CleanImport(name):64        with extension_redirect(name, path) as new_path:65            with import_helper.DirsOnSysPath(new_path):66                yield67 68 69@contextlib.contextmanager70def extension_redirect(mod, path):71    """72    Tests will fail to tear down an extension module if it's been imported.73 74    Before importing, copy the file to a temporary directory that won't75    be cleaned up. Yield the new path.76    """77    if platform.system() != "Windows" and sys.platform != "cygwin":78        yield path79        return80    with import_helper.DirsOnSysPath(path):81        spec = importlib.util.find_spec(mod)82    filename = os.path.basename(spec.origin)83    trash_dir = tempfile.mkdtemp(prefix='deleteme')84    dest = os.path.join(trash_dir, os.path.basename(filename))85    shutil.copy(spec.origin, dest)86    yield trash_dir87    # TODO: can the file be scheduled for deletion?88 89 90@pytest.mark.usefixtures('user_site_dir')91class TestBuildExt(TempdirManager):92    def build_ext(self, *args, **kwargs):93        return build_ext(*args, **kwargs)94 95    @pytest.mark.parametrize("copy_so", [False])96    def test_build_ext(self, copy_so):97        missing_compiler_executable()98        copy_xxmodule_c(self.tmp_dir)99        xx_c = os.path.join(self.tmp_dir, 'xxmodule.c')100        xx_ext = Extension('xx', [xx_c])101        if sys.platform != "win32":102            if not copy_so:103                xx_ext = Extension(104                    'xx',105                    [xx_c],106                    library_dirs=['/usr/lib'],107                    libraries=['z'],108                    runtime_library_dirs=['/usr/lib'],109                )110            elif sys.platform == 'linux':111                libz_so = {112                    os.path.realpath(name) for name in glob.iglob('/usr/lib*/libz.so*')113                }114                libz_so = sorted(libz_so, key=lambda lib_path: len(lib_path))115                shutil.copyfile(libz_so[-1], '/tmp/libxx_z.so')116 117                xx_ext = Extension(118                    'xx',119                    [xx_c],120                    library_dirs=['/tmp'],121                    libraries=['xx_z'],122                    runtime_library_dirs=['/tmp'],123                )124        dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]})125        dist.package_dir = self.tmp_dir126        cmd = self.build_ext(dist)127        fixup_build_ext(cmd)128        cmd.build_lib = self.tmp_dir129        cmd.build_temp = self.tmp_dir130 131        old_stdout = sys.stdout132        if not support.verbose:133            # silence compiler output134            sys.stdout = StringIO()135        try:136            cmd.ensure_finalized()137            cmd.run()138        finally:139            sys.stdout = old_stdout140 141        with safe_extension_import('xx', self.tmp_dir):142            self._test_xx(copy_so)143 144        if sys.platform == 'linux' and copy_so:145            os.unlink('/tmp/libxx_z.so')146 147    @staticmethod148    def _test_xx(copy_so):149        import xx  # type: ignore[import-not-found] # Module generated for tests150 151        for attr in ('error', 'foo', 'new', 'roj'):152            assert hasattr(xx, attr)153 154        assert xx.foo(2, 5) == 7155        assert xx.foo(13, 15) == 28156        assert xx.new().demo() is None157        if support.HAVE_DOCSTRINGS:158            doc = 'This is a template module just for instruction.'159            assert xx.__doc__ == doc160        assert isinstance(xx.Null(), xx.Null)161        assert isinstance(xx.Str(), xx.Str)162 163        if sys.platform == 'linux':164            so_headers = subprocess.check_output(165                ["readelf", "-d", xx.__file__], universal_newlines=True166            )167            import pprint168 169            pprint.pprint(so_headers)170            rpaths = [171                rpath172                for line in so_headers.split("\n")173                if "RPATH" in line or "RUNPATH" in line174                for rpath in line.split()[2][1:-1].split(":")175            ]176            if not copy_so:177                pprint.pprint(rpaths)178                # Linked against a library in /usr/lib{,64}179                assert "/usr/lib" not in rpaths and "/usr/lib64" not in rpaths180            else:181                # Linked against a library in /tmp182                assert "/tmp" in rpaths183                # The import is the real test here184 185    def test_solaris_enable_shared(self):186        dist = Distribution({'name': 'xx'})187        cmd = self.build_ext(dist)188        old = sys.platform189 190        sys.platform = 'sunos'  # fooling finalize_options191        from distutils.sysconfig import _config_vars192 193        old_var = _config_vars.get('Py_ENABLE_SHARED')194        _config_vars['Py_ENABLE_SHARED'] = True195        try:196            cmd.ensure_finalized()197        finally:198            sys.platform = old199            if old_var is None:200                del _config_vars['Py_ENABLE_SHARED']201            else:202                _config_vars['Py_ENABLE_SHARED'] = old_var203 204        # make sure we get some library dirs under solaris205        assert len(cmd.library_dirs) > 0206 207    def test_user_site(self):208        import site209 210        dist = Distribution({'name': 'xx'})211        cmd = self.build_ext(dist)212 213        # making sure the user option is there214        options = [name for name, short, label in cmd.user_options]215        assert 'user' in options216 217        # setting a value218        cmd.user = True219 220        # setting user based lib and include221        lib = os.path.join(site.USER_BASE, 'lib')222        incl = os.path.join(site.USER_BASE, 'include')223        os.mkdir(lib)224        os.mkdir(incl)225 226        # let's run finalize227        cmd.ensure_finalized()228 229        # see if include_dirs and library_dirs230        # were set231        assert lib in cmd.library_dirs232        assert lib in cmd.rpath233        assert incl in cmd.include_dirs234 235    def test_optional_extension(self):236        # this extension will fail, but let's ignore this failure237        # with the optional argument.238        modules = [Extension('foo', ['xxx'], optional=False)]239        dist = Distribution({'name': 'xx', 'ext_modules': modules})240        cmd = self.build_ext(dist)241        cmd.ensure_finalized()242        with pytest.raises((UnknownFileError, CompileError)):243            cmd.run()  # should raise an error244 245        modules = [Extension('foo', ['xxx'], optional=True)]246        dist = Distribution({'name': 'xx', 'ext_modules': modules})247        cmd = self.build_ext(dist)248        cmd.ensure_finalized()249        cmd.run()  # should pass250 251    def test_finalize_options(self):252        # Make sure Python's include directories (for Python.h, pyconfig.h,253        # etc.) are in the include search path.254        modules = [Extension('foo', ['xxx'], optional=False)]255        dist = Distribution({'name': 'xx', 'ext_modules': modules})256        cmd = self.build_ext(dist)257        cmd.finalize_options()258 259        py_include = sysconfig.get_python_inc()260        for p in py_include.split(os.path.pathsep):261            assert p in cmd.include_dirs262 263        plat_py_include = sysconfig.get_python_inc(plat_specific=True)264        for p in plat_py_include.split(os.path.pathsep):265            assert p in cmd.include_dirs266 267        # make sure cmd.libraries is turned into a list268        # if it's a string269        cmd = self.build_ext(dist)270        cmd.libraries = 'my_lib, other_lib lastlib'271        cmd.finalize_options()272        assert cmd.libraries == ['my_lib', 'other_lib', 'lastlib']273 274        # make sure cmd.library_dirs is turned into a list275        # if it's a string276        cmd = self.build_ext(dist)277        cmd.library_dirs = f'my_lib_dir{os.pathsep}other_lib_dir'278        cmd.finalize_options()279        assert 'my_lib_dir' in cmd.library_dirs280        assert 'other_lib_dir' in cmd.library_dirs281 282        # make sure rpath is turned into a list283        # if it's a string284        cmd = self.build_ext(dist)285        cmd.rpath = f'one{os.pathsep}two'286        cmd.finalize_options()287        assert cmd.rpath == ['one', 'two']288 289        # make sure cmd.link_objects is turned into a list290        # if it's a string291        cmd = build_ext(dist)292        cmd.link_objects = 'one two,three'293        cmd.finalize_options()294        assert cmd.link_objects == ['one', 'two', 'three']295 296        # XXX more tests to perform for win32297 298        # make sure define is turned into 2-tuples299        # strings if they are ','-separated strings300        cmd = self.build_ext(dist)301        cmd.define = 'one,two'302        cmd.finalize_options()303        assert cmd.define == [('one', '1'), ('two', '1')]304 305        # make sure undef is turned into a list of306        # strings if they are ','-separated strings307        cmd = self.build_ext(dist)308        cmd.undef = 'one,two'309        cmd.finalize_options()310        assert cmd.undef == ['one', 'two']311 312        # make sure swig_opts is turned into a list313        cmd = self.build_ext(dist)314        cmd.swig_opts = None315        cmd.finalize_options()316        assert cmd.swig_opts == []317 318        cmd = self.build_ext(dist)319        cmd.swig_opts = '1 2'320        cmd.finalize_options()321        assert cmd.swig_opts == ['1', '2']322 323    def test_check_extensions_list(self):324        dist = Distribution()325        cmd = self.build_ext(dist)326        cmd.finalize_options()327 328        # 'extensions' option must be a list of Extension instances329        with pytest.raises(DistutilsSetupError):330            cmd.check_extensions_list('foo')331 332        # each element of 'ext_modules' option must be an333        # Extension instance or 2-tuple334        exts = [('bar', 'foo', 'bar'), 'foo']335        with pytest.raises(DistutilsSetupError):336            cmd.check_extensions_list(exts)337 338        # first element of each tuple in 'ext_modules'339        # must be the extension name (a string) and match340        # a python dotted-separated name341        exts = [('foo-bar', '')]342        with pytest.raises(DistutilsSetupError):343            cmd.check_extensions_list(exts)344 345        # second element of each tuple in 'ext_modules'346        # must be a dictionary (build info)347        exts = [('foo.bar', '')]348        with pytest.raises(DistutilsSetupError):349            cmd.check_extensions_list(exts)350 351        # ok this one should pass352        exts = [('foo.bar', {'sources': [''], 'libraries': 'foo', 'some': 'bar'})]353        cmd.check_extensions_list(exts)354        ext = exts[0]355        assert isinstance(ext, Extension)356 357        # check_extensions_list adds in ext the values passed358        # when they are in ('include_dirs', 'library_dirs', 'libraries'359        # 'extra_objects', 'extra_compile_args', 'extra_link_args')360        assert ext.libraries == 'foo'361        assert not hasattr(ext, 'some')362 363        # 'macros' element of build info dict must be 1- or 2-tuple364        exts = [365            (366                'foo.bar',367                {368                    'sources': [''],369                    'libraries': 'foo',370                    'some': 'bar',371                    'macros': [('1', '2', '3'), 'foo'],372                },373            )374        ]375        with pytest.raises(DistutilsSetupError):376            cmd.check_extensions_list(exts)377 378        exts[0][1]['macros'] = [('1', '2'), ('3',)]379        cmd.check_extensions_list(exts)380        assert exts[0].undef_macros == ['3']381        assert exts[0].define_macros == [('1', '2')]382 383    def test_get_source_files(self):384        modules = [Extension('foo', ['xxx'], optional=False)]385        dist = Distribution({'name': 'xx', 'ext_modules': modules})386        cmd = self.build_ext(dist)387        cmd.ensure_finalized()388        assert cmd.get_source_files() == ['xxx']389 390    def test_unicode_module_names(self):391        modules = [392            Extension('foo', ['aaa'], optional=False),393            Extension('föö', ['uuu'], optional=False),394        ]395        dist = Distribution({'name': 'xx', 'ext_modules': modules})396        cmd = self.build_ext(dist)397        cmd.ensure_finalized()398        assert re.search(r'foo(_d)?\..*', cmd.get_ext_filename(modules[0].name))399        assert re.search(r'föö(_d)?\..*', cmd.get_ext_filename(modules[1].name))400        assert cmd.get_export_symbols(modules[0]) == ['PyInit_foo']401        assert cmd.get_export_symbols(modules[1]) == ['PyInitU_f_1gaa']402 403    def test_export_symbols__init__(self):404        # https://github.com/python/cpython/issues/80074405        # https://github.com/pypa/setuptools/issues/4826406        modules = [407            Extension('foo.__init__', ['aaa']),408            Extension('föö.__init__', ['uuu']),409        ]410        dist = Distribution({'name': 'xx', 'ext_modules': modules})411        cmd = self.build_ext(dist)412        cmd.ensure_finalized()413        assert cmd.get_export_symbols(modules[0]) == ['PyInit_foo']414        assert cmd.get_export_symbols(modules[1]) == ['PyInitU_f_1gaa']415 416    def test_compiler_option(self):417        # cmd.compiler is an option and418        # should not be overridden by a compiler instance419        # when the command is run420        dist = Distribution()421        cmd = self.build_ext(dist)422        cmd.compiler = 'unix'423        cmd.ensure_finalized()424        cmd.run()425        assert cmd.compiler == 'unix'426 427    def test_get_outputs(self):428        missing_compiler_executable()429        tmp_dir = self.mkdtemp()430        c_file = os.path.join(tmp_dir, 'foo.c')431        self.write_file(c_file, 'void PyInit_foo(void) {}\n')432        ext = Extension('foo', [c_file], optional=False)433        dist = Distribution({'name': 'xx', 'ext_modules': [ext]})434        cmd = self.build_ext(dist)435        fixup_build_ext(cmd)436        cmd.ensure_finalized()437        assert len(cmd.get_outputs()) == 1438 439        cmd.build_lib = os.path.join(self.tmp_dir, 'build')440        cmd.build_temp = os.path.join(self.tmp_dir, 'tempt')441 442        # issue #5977 : distutils build_ext.get_outputs443        # returns wrong result with --inplace444        other_tmp_dir = os.path.realpath(self.mkdtemp())445        old_wd = os.getcwd()446        os.chdir(other_tmp_dir)447        try:448            cmd.inplace = True449            cmd.run()450            so_file = cmd.get_outputs()[0]451        finally:452            os.chdir(old_wd)453        assert os.path.exists(so_file)454        ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')455        assert so_file.endswith(ext_suffix)456        so_dir = os.path.dirname(so_file)457        assert so_dir == other_tmp_dir458 459        cmd.inplace = False460        cmd.compiler = None461        cmd.run()462        so_file = cmd.get_outputs()[0]463        assert os.path.exists(so_file)464        assert so_file.endswith(ext_suffix)465        so_dir = os.path.dirname(so_file)466        assert so_dir == cmd.build_lib467 468        # inplace = False, cmd.package = 'bar'469        build_py = cmd.get_finalized_command('build_py')470        build_py.package_dir = {'': 'bar'}471        path = cmd.get_ext_fullpath('foo')472        # checking that the last directory is the build_dir473        path = os.path.split(path)[0]474        assert path == cmd.build_lib475 476        # inplace = True, cmd.package = 'bar'477        cmd.inplace = True478        other_tmp_dir = os.path.realpath(self.mkdtemp())479        old_wd = os.getcwd()480        os.chdir(other_tmp_dir)481        try:482            path = cmd.get_ext_fullpath('foo')483        finally:484            os.chdir(old_wd)485        # checking that the last directory is bar486        path = os.path.split(path)[0]487        lastdir = os.path.split(path)[-1]488        assert lastdir == 'bar'489 490    def test_ext_fullpath(self):491        ext = sysconfig.get_config_var('EXT_SUFFIX')492        # building lxml.etree inplace493        # etree_c = os.path.join(self.tmp_dir, 'lxml.etree.c')494        # etree_ext = Extension('lxml.etree', [etree_c])495        # dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]})496        dist = Distribution()497        cmd = self.build_ext(dist)498        cmd.inplace = True499        cmd.distribution.package_dir = {'': 'src'}500        cmd.distribution.packages = ['lxml', 'lxml.html']501        curdir = os.getcwd()502        wanted = os.path.join(curdir, 'src', 'lxml', 'etree' + ext)503        path = cmd.get_ext_fullpath('lxml.etree')504        assert wanted == path505 506        # building lxml.etree not inplace507        cmd.inplace = False508        cmd.build_lib = os.path.join(curdir, 'tmpdir')509        wanted = os.path.join(curdir, 'tmpdir', 'lxml', 'etree' + ext)510        path = cmd.get_ext_fullpath('lxml.etree')511        assert wanted == path512 513        # building twisted.runner.portmap not inplace514        build_py = cmd.get_finalized_command('build_py')515        build_py.package_dir = {}516        cmd.distribution.packages = ['twisted', 'twisted.runner.portmap']517        path = cmd.get_ext_fullpath('twisted.runner.portmap')518        wanted = os.path.join(curdir, 'tmpdir', 'twisted', 'runner', 'portmap' + ext)519        assert wanted == path520 521        # building twisted.runner.portmap inplace522        cmd.inplace = True523        path = cmd.get_ext_fullpath('twisted.runner.portmap')524        wanted = os.path.join(curdir, 'twisted', 'runner', 'portmap' + ext)525        assert wanted == path526 527    @pytest.mark.skipif('platform.system() != "Darwin"')528    @pytest.mark.usefixtures('save_env')529    def test_deployment_target_default(self):530        # Issue 9516: Test that, in the absence of the environment variable,531        # an extension module is compiled with the same deployment target as532        #  the interpreter.533        self._try_compile_deployment_target('==', None)534 535    @pytest.mark.skipif('platform.system() != "Darwin"')536    @pytest.mark.usefixtures('save_env')537    def test_deployment_target_too_low(self):538        # Issue 9516: Test that an extension module is not allowed to be539        # compiled with a deployment target less than that of the interpreter.540        with pytest.raises(DistutilsPlatformError):541            self._try_compile_deployment_target('>', '10.1')542 543    @pytest.mark.skipif('platform.system() != "Darwin"')544    @pytest.mark.usefixtures('save_env')545    def test_deployment_target_higher_ok(self):  # pragma: no cover546        # Issue 9516: Test that an extension module can be compiled with a547        # deployment target higher than that of the interpreter: the ext548        # module may depend on some newer OS feature.549        deptarget = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')550        if deptarget:551            # increment the minor version number (i.e. 10.6 -> 10.7)552            deptarget = [int(x) for x in deptarget.split('.')]553            deptarget[-1] += 1554            deptarget = '.'.join(str(i) for i in deptarget)555            self._try_compile_deployment_target('<', deptarget)556 557    def _try_compile_deployment_target(self, operator, target):  # pragma: no cover558        if target is None:559            if os.environ.get('MACOSX_DEPLOYMENT_TARGET'):560                del os.environ['MACOSX_DEPLOYMENT_TARGET']561        else:562            os.environ['MACOSX_DEPLOYMENT_TARGET'] = target563 564        jaraco.path.build(565            {566                'deptargetmodule.c': textwrap.dedent(f"""\567                    #include <AvailabilityMacros.h>568 569                    int dummy;570 571                    #if TARGET {operator} MAC_OS_X_VERSION_MIN_REQUIRED572                    #else573                    #error "Unexpected target"574                    #endif575 576                    """),577            },578            self.tmp_path,579        )580 581        # get the deployment target that the interpreter was built with582        target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')583        target = tuple(map(int, target.split('.')[0:2]))584        # format the target value as defined in the Apple585        # Availability Macros.  We can't use the macro names since586        # at least one value we test with will not exist yet.587        if target[:2] < (10, 10):588            # for 10.1 through 10.9.x -> "10n0"589            tmpl = '{:02}{:01}0'590        else:591            # for 10.10 and beyond -> "10nn00"592            if len(target) >= 2:593                tmpl = '{:02}{:02}00'594            else:595                # 11 and later can have no minor version (11 instead of 11.0)596                tmpl = '{:02}0000'597        target = tmpl.format(*target)598        deptarget_ext = Extension(599            'deptarget',600            [self.tmp_path / 'deptargetmodule.c'],601            extra_compile_args=[f'-DTARGET={target}'],602        )603        dist = Distribution({'name': 'deptarget', 'ext_modules': [deptarget_ext]})604        dist.package_dir = self.tmp_dir605        cmd = self.build_ext(dist)606        cmd.build_lib = self.tmp_dir607        cmd.build_temp = self.tmp_dir608 609        try:610            old_stdout = sys.stdout611            if not support.verbose:612                # silence compiler output613                sys.stdout = StringIO()614            try:615                cmd.ensure_finalized()616                cmd.run()617            finally:618                sys.stdout = old_stdout619 620        except CompileError:621            self.fail("Wrong deployment target during compilation")622 623 624class TestParallelBuildExt(TestBuildExt):625    def build_ext(self, *args, **kwargs):626        build_ext = super().build_ext(*args, **kwargs)627        build_ext.parallel = True628        return build_ext629