codekingpro/portable-devtools

1import os2import sys3from configparser import ConfigParser4from itertools import product5from typing import cast6 7import jaraco.path8import pytest9from path import Path10 11import setuptools  # noqa: F401 # force distutils.core to be patched12from setuptools.command.sdist import sdist13from setuptools.discovery import find_package_path, find_parent_package14from setuptools.dist import Distribution15from setuptools.errors import PackageDiscoveryError16 17from .contexts import quiet18from .integration.helpers import get_sdist_members, get_wheel_members, run19from .textwrap import DALS20 21import distutils.core22 23 24class TestFindParentPackage:25    def test_single_package(self, tmp_path):26        # find_parent_package should find a non-namespace parent package27        (tmp_path / "src/namespace/pkg/nested").mkdir(exist_ok=True, parents=True)28        (tmp_path / "src/namespace/pkg/nested/__init__.py").touch()29        (tmp_path / "src/namespace/pkg/__init__.py").touch()30        packages = ["namespace", "namespace.pkg", "namespace.pkg.nested"]31        assert find_parent_package(packages, {"": "src"}, tmp_path) == "namespace.pkg"32 33    def test_multiple_toplevel(self, tmp_path):34        # find_parent_package should return null if the given list of packages does not35        # have a single parent package36        multiple = ["pkg", "pkg1", "pkg2"]37        for name in multiple:38            (tmp_path / f"src/{name}").mkdir(exist_ok=True, parents=True)39            (tmp_path / f"src/{name}/__init__.py").touch()40        assert find_parent_package(multiple, {"": "src"}, tmp_path) is None41 42 43class TestDiscoverPackagesAndPyModules:44    """Make sure discovered values for ``packages`` and ``py_modules`` work45    similarly to explicit configuration for the simple scenarios.46    """47 48    OPTIONS = {49        # Different options according to the circumstance being tested50        "explicit-src": {"package_dir": {"": "src"}, "packages": ["pkg"]},51        "variation-lib": {52            "package_dir": {"": "lib"},  # variation of the source-layout53        },54        "explicit-flat": {"packages": ["pkg"]},55        "explicit-single_module": {"py_modules": ["pkg"]},56        "explicit-namespace": {"packages": ["ns", "ns.pkg"]},57        "automatic-src": {},58        "automatic-flat": {},59        "automatic-single_module": {},60        "automatic-namespace": {},61    }62    FILES = {63        "src": ["src/pkg/__init__.py", "src/pkg/main.py"],64        "lib": ["lib/pkg/__init__.py", "lib/pkg/main.py"],65        "flat": ["pkg/__init__.py", "pkg/main.py"],66        "single_module": ["pkg.py"],67        "namespace": ["ns/pkg/__init__.py"],68    }69 70    def _get_info(self, circumstance):71        _, _, layout = circumstance.partition("-")72        files = self.FILES[layout]73        options = self.OPTIONS[circumstance]74        return files, options75 76    @pytest.mark.parametrize("circumstance", OPTIONS.keys())77    def test_sdist_filelist(self, tmp_path, circumstance):78        files, options = self._get_info(circumstance)79        _populate_project_dir(tmp_path, files, options)80 81        _, cmd = _run_sdist_programatically(tmp_path, options)82 83        manifest = [f.replace(os.sep, "/") for f in cmd.filelist.files]84        for file in files:85            assert any(f.endswith(file) for f in manifest)86 87    @pytest.mark.parametrize("circumstance", OPTIONS.keys())88    def test_project(self, tmp_path, circumstance):89        files, options = self._get_info(circumstance)90        _populate_project_dir(tmp_path, files, options)91 92        # Simulate a pre-existing `build` directory93        (tmp_path / "build").mkdir()94        (tmp_path / "build/lib").mkdir()95        (tmp_path / "build/bdist.linux-x86_64").mkdir()96        (tmp_path / "build/bdist.linux-x86_64/file.py").touch()97        (tmp_path / "build/lib/__init__.py").touch()98        (tmp_path / "build/lib/file.py").touch()99        (tmp_path / "dist").mkdir()100        (tmp_path / "dist/file.py").touch()101 102        _run_build(tmp_path)103 104        sdist_files = get_sdist_members(next(tmp_path.glob("dist/*.tar.gz")))105        print("~~~~~ sdist_members ~~~~~")106        print('\n'.join(sdist_files))107        assert sdist_files >= set(files)108 109        wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl")))110        print("~~~~~ wheel_members ~~~~~")111        print('\n'.join(wheel_files))112        orig_files = {f.replace("src/", "").replace("lib/", "") for f in files}113        assert wheel_files >= orig_files114 115        # Make sure build files are not included by mistake116        for file in wheel_files:117            assert "build" not in files118            assert "dist" not in files119 120    PURPOSEFULLY_EMPY = {121        "setup.cfg": DALS(122            """123            [metadata]124            name = myproj125            version = 0.0.0126 127            [options]128            {param} =129            """130        ),131        "setup.py": DALS(132            """133            __import__('setuptools').setup(134                name="myproj",135                version="0.0.0",136                {param}=[]137            )138            """139        ),140        "pyproject.toml": DALS(141            """142            [build-system]143            requires = []144            build-backend = 'setuptools.build_meta'145 146            [project]147            name = "myproj"148            version = "0.0.0"149 150            [tool.setuptools]151            {param} = []152            """153        ),154        "template-pyproject.toml": DALS(155            """156            [build-system]157            requires = []158            build-backend = 'setuptools.build_meta'159            """160        ),161    }162 163    @pytest.mark.parametrize(164        ("config_file", "param", "circumstance"),165        product(166            ["setup.cfg", "setup.py", "pyproject.toml"],167            ["packages", "py_modules"],168            FILES.keys(),169        ),170    )171    def test_purposefully_empty(self, tmp_path, config_file, param, circumstance):172        files = self.FILES[circumstance] + ["mod.py", "other.py", "src/pkg/__init__.py"]173        _populate_project_dir(tmp_path, files, {})174 175        if config_file == "pyproject.toml":176            template_param = param.replace("_", "-")177        else:178            # Make sure build works with or without setup.cfg179            pyproject = self.PURPOSEFULLY_EMPY["template-pyproject.toml"]180            (tmp_path / "pyproject.toml").write_text(pyproject, encoding="utf-8")181            template_param = param182 183        config = self.PURPOSEFULLY_EMPY[config_file].format(param=template_param)184        (tmp_path / config_file).write_text(config, encoding="utf-8")185 186        dist = _get_dist(tmp_path, {})187        # When either parameter package or py_modules is an empty list,188        # then there should be no discovery189        assert getattr(dist, param) == []190        other = {"py_modules": "packages", "packages": "py_modules"}[param]191        assert getattr(dist, other) is None192 193    @pytest.mark.parametrize(194        ("extra_files", "pkgs"),195        [196            (["venv/bin/simulate_venv"], {"pkg"}),197            (["pkg-stubs/__init__.pyi"], {"pkg", "pkg-stubs"}),198            (["other-stubs/__init__.pyi"], {"pkg", "other-stubs"}),199            (200                # Type stubs can also be namespaced201                ["namespace-stubs/pkg/__init__.pyi"],202                {"pkg", "namespace-stubs", "namespace-stubs.pkg"},203            ),204            (205                # Just the top-level package can have `-stubs`, ignore nested ones206                ["namespace-stubs/pkg-stubs/__init__.pyi"],207                {"pkg", "namespace-stubs"},208            ),209            (["_hidden/file.py"], {"pkg"}),210            (["news/finalize.py"], {"pkg"}),211        ],212    )213    def test_flat_layout_with_extra_files(self, tmp_path, extra_files, pkgs):214        files = self.FILES["flat"] + extra_files215        _populate_project_dir(tmp_path, files, {})216        dist = _get_dist(tmp_path, {})217        assert set(dist.packages) == pkgs218 219    @pytest.mark.parametrize(220        "extra_files",221        [222            ["other/__init__.py"],223            ["other/finalize.py"],224        ],225    )226    def test_flat_layout_with_dangerous_extra_files(self, tmp_path, extra_files):227        files = self.FILES["flat"] + extra_files228        _populate_project_dir(tmp_path, files, {})229        with pytest.raises(PackageDiscoveryError, match="multiple (packages|modules)"):230            _get_dist(tmp_path, {})231 232    def test_flat_layout_with_single_module(self, tmp_path):233        files = self.FILES["single_module"] + ["invalid-module-name.py"]234        _populate_project_dir(tmp_path, files, {})235        dist = _get_dist(tmp_path, {})236        assert set(dist.py_modules) == {"pkg"}237 238    def test_flat_layout_with_multiple_modules(self, tmp_path):239        files = self.FILES["single_module"] + ["valid_module_name.py"]240        _populate_project_dir(tmp_path, files, {})241        with pytest.raises(PackageDiscoveryError, match="multiple (packages|modules)"):242            _get_dist(tmp_path, {})243 244    def test_py_modules_when_wheel_dir_is_cwd(self, tmp_path):245        """Regression for issue 3692"""246        from setuptools import build_meta247 248        pyproject = '[project]\nname = "test"\nversion = "1"'249        (tmp_path / "pyproject.toml").write_text(DALS(pyproject), encoding="utf-8")250        (tmp_path / "foo.py").touch()251        with jaraco.path.DirectoryStack().context(tmp_path):252            build_meta.build_wheel(".")253        # Ensure py_modules are found254        wheel_files = get_wheel_members(next(tmp_path.glob("*.whl")))255        assert "foo.py" in wheel_files256 257 258class TestNoConfig:259    DEFAULT_VERSION = "0.0.0"  # Default version given by setuptools260 261    EXAMPLES = {262        "pkg1": ["src/pkg1.py"],263        "pkg2": ["src/pkg2/__init__.py"],264        "pkg3": ["src/pkg3/__init__.py", "src/pkg3-stubs/__init__.py"],265        "pkg4": ["pkg4/__init__.py", "pkg4-stubs/__init__.py"],266        "ns.nested.pkg1": ["src/ns/nested/pkg1/__init__.py"],267        "ns.nested.pkg2": ["ns/nested/pkg2/__init__.py"],268    }269 270    @pytest.mark.parametrize("example", EXAMPLES.keys())271    def test_discover_name(self, tmp_path, example):272        _populate_project_dir(tmp_path, self.EXAMPLES[example], {})273        dist = _get_dist(tmp_path, {})274        assert dist.get_name() == example275 276    def test_build_with_discovered_name(self, tmp_path):277        files = ["src/ns/nested/pkg/__init__.py"]278        _populate_project_dir(tmp_path, files, {})279        _run_build(tmp_path, "--sdist")280        # Expected distribution file281        dist_file = tmp_path / f"dist/ns_nested_pkg-{self.DEFAULT_VERSION}.tar.gz"282        assert dist_file.is_file()283 284 285class TestWithAttrDirective:286    @pytest.mark.parametrize(287        ("folder", "opts"),288        [289            ("src", {}),290            ("lib", {"packages": "find:", "packages.find": {"where": "lib"}}),291        ],292    )293    def test_setupcfg_metadata(self, tmp_path, folder, opts):294        files = [f"{folder}/pkg/__init__.py", "setup.cfg"]295        _populate_project_dir(tmp_path, files, opts)296 297        config = (tmp_path / "setup.cfg").read_text(encoding="utf-8")298        overwrite = {299            folder: {"pkg": {"__init__.py": "version = 42"}},300            "setup.cfg": "[metadata]\nversion = attr: pkg.version\n" + config,301        }302        jaraco.path.build(overwrite, prefix=tmp_path)303 304        dist = _get_dist(tmp_path, {})305        assert dist.get_name() == "pkg"306        assert dist.get_version() == "42"307        assert dist.package_dir308        package_path = find_package_path("pkg", dist.package_dir, tmp_path)309        assert os.path.exists(package_path)310        assert folder in Path(package_path).parts()311 312        _run_build(tmp_path, "--sdist")313        dist_file = tmp_path / "dist/pkg-42.tar.gz"314        assert dist_file.is_file()315 316    def test_pyproject_metadata(self, tmp_path):317        _populate_project_dir(tmp_path, ["src/pkg/__init__.py"], {})318 319        overwrite = {320            "src": {"pkg": {"__init__.py": "version = 42"}},321            "pyproject.toml": (322                "[project]\nname = 'pkg'\ndynamic = ['version']\n"323                "[tool.setuptools.dynamic]\nversion = {attr = 'pkg.version'}\n"324            ),325        }326        jaraco.path.build(overwrite, prefix=tmp_path)327 328        dist = _get_dist(tmp_path, {})329        assert dist.get_version() == "42"330        assert dist.package_dir == {"": "src"}331 332 333class TestWithCExtension:334    def _simulate_package_with_extension(self, tmp_path):335        # This example is based on: https://github.com/nucleic/kiwi/tree/1.4.0336        files = [337            "benchmarks/file.py",338            "docs/Makefile",339            "docs/requirements.txt",340            "docs/source/conf.py",341            "proj/header.h",342            "proj/file.py",343            "py/proj.cpp",344            "py/other.cpp",345            "py/file.py",346            "py/py.typed",347            "py/tests/test_proj.py",348            "README.rst",349        ]350        _populate_project_dir(tmp_path, files, {})351 352        setup_script = """353            from setuptools import Extension, setup354 355            ext_modules = [356                Extension(357                    "proj",358                    ["py/proj.cpp", "py/other.cpp"],359                    include_dirs=["."],360                    language="c++",361                ),362            ]363            setup(ext_modules=ext_modules)364        """365        (tmp_path / "setup.py").write_text(DALS(setup_script), encoding="utf-8")366 367    def test_skip_discovery_with_setupcfg_metadata(self, tmp_path):368        """Ensure that auto-discovery is not triggered when the project is based on369        C-extensions only, for backward compatibility.370        """371        self._simulate_package_with_extension(tmp_path)372 373        pyproject = """374            [build-system]375            requires = []376            build-backend = 'setuptools.build_meta'377        """378        (tmp_path / "pyproject.toml").write_text(DALS(pyproject), encoding="utf-8")379 380        setupcfg = """381            [metadata]382            name = proj383            version = 42384        """385        (tmp_path / "setup.cfg").write_text(DALS(setupcfg), encoding="utf-8")386 387        dist = _get_dist(tmp_path, {})388        assert dist.get_name() == "proj"389        assert dist.get_version() == "42"390        assert dist.py_modules is None391        assert dist.packages is None392        assert len(dist.ext_modules) == 1393        assert dist.ext_modules[0].name == "proj"394 395    def test_dont_skip_discovery_with_pyproject_metadata(self, tmp_path):396        """When opting-in to pyproject.toml metadata, auto-discovery will be active if397        the package lists C-extensions, but does not configure py-modules or packages.398 399        This way we ensure users with complex package layouts that would lead to the400        discovery of multiple top-level modules/packages see errors and are forced to401        explicitly set ``packages`` or ``py-modules``.402        """403        self._simulate_package_with_extension(tmp_path)404 405        pyproject = """406            [project]407            name = 'proj'408            version = '42'409        """410        (tmp_path / "pyproject.toml").write_text(DALS(pyproject), encoding="utf-8")411        with pytest.raises(PackageDiscoveryError, match="multiple (packages|modules)"):412            _get_dist(tmp_path, {})413 414 415class TestWithPackageData:416    def _simulate_package_with_data_files(self, tmp_path, src_root):417        files = [418            f"{src_root}/proj/__init__.py",419            f"{src_root}/proj/file1.txt",420            f"{src_root}/proj/nested/file2.txt",421        ]422        _populate_project_dir(tmp_path, files, {})423 424        manifest = """425            global-include *.py *.txt426        """427        (tmp_path / "MANIFEST.in").write_text(DALS(manifest), encoding="utf-8")428 429    EXAMPLE_SETUPCFG = """430    [metadata]431    name = proj432    version = 42433 434    [options]435    include_package_data = True436    """437    EXAMPLE_PYPROJECT = """438    [project]439    name = "proj"440    version = "42"441    """442 443    PYPROJECT_PACKAGE_DIR = """444    [tool.setuptools]445    package-dir = {"" = "src"}446    """447 448    @pytest.mark.parametrize(449        ("src_root", "files"),450        [451            (".", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}),452            (".", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}),453            ("src", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}),454            ("src", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}),455            (456                "src",457                {458                    "setup.cfg": DALS(EXAMPLE_SETUPCFG)459                    + DALS(460                        """461                        packages = find:462                        package_dir =463                            =src464 465                        [options.packages.find]466                        where = src467                        """468                    )469                },470            ),471            (472                "src",473                {474                    "pyproject.toml": DALS(EXAMPLE_PYPROJECT)475                    + DALS(476                        """477                        [tool.setuptools]478                        package-dir = {"" = "src"}479                        """480                    )481                },482            ),483        ],484    )485    def test_include_package_data(self, tmp_path, src_root, files):486        """487        Make sure auto-discovery does not affect package include_package_data.488        See issue #3196.489        """490        jaraco.path.build(files, prefix=str(tmp_path))491        self._simulate_package_with_data_files(tmp_path, src_root)492 493        expected = {494            os.path.normpath(f"{src_root}/proj/file1.txt").replace(os.sep, "/"),495            os.path.normpath(f"{src_root}/proj/nested/file2.txt").replace(os.sep, "/"),496        }497 498        _run_build(tmp_path)499 500        sdist_files = get_sdist_members(next(tmp_path.glob("dist/*.tar.gz")))501        print("~~~~~ sdist_members ~~~~~")502        print('\n'.join(sdist_files))503        assert sdist_files >= expected504 505        wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl")))506        print("~~~~~ wheel_members ~~~~~")507        print('\n'.join(wheel_files))508        orig_files = {f.replace("src/", "").replace("lib/", "") for f in expected}509        assert wheel_files >= orig_files510 511 512def test_compatible_with_numpy_configuration(tmp_path):513    files = [514        "dir1/__init__.py",515        "dir2/__init__.py",516        "file.py",517    ]518    _populate_project_dir(tmp_path, files, {})519    dist = Distribution({})520    dist.configuration = object()521    dist.set_defaults()522    assert dist.py_modules is None523    assert dist.packages is None524 525 526def test_name_discovery_doesnt_break_cli(tmpdir_cwd):527    jaraco.path.build({"pkg.py": ""})528    dist = Distribution({})529    dist.script_args = ["--name"]530    dist.set_defaults()531    dist.parse_command_line()  # <-- no exception should be raised here.532    assert dist.get_name() == "pkg"533 534 535def test_preserve_explicit_name_with_dynamic_version(tmpdir_cwd, monkeypatch):536    """According to #3545 it seems that ``name`` discovery is running,537    even when the project already explicitly sets it.538    This seems to be related to parsing of dynamic versions (via ``attr`` directive),539    which requires the auto-discovery of ``package_dir``.540    """541    files = {542        "src": {543            "pkg": {"__init__.py": "__version__ = 42\n"},544        },545        "pyproject.toml": DALS(546            """547            [project]548            name = "myproj"  # purposefully different from package name549            dynamic = ["version"]550            [tool.setuptools.dynamic]551            version = {"attr" = "pkg.__version__"}552            """553        ),554    }555    jaraco.path.build(files)556    dist = Distribution({})557    orig_analyse_name = dist.set_defaults.analyse_name558 559    def spy_analyse_name():560        # We can check if name discovery was triggered by ensuring the original561        # name remains instead of the package name.562        orig_analyse_name()563        assert dist.get_name() == "myproj"564 565    monkeypatch.setattr(dist.set_defaults, "analyse_name", spy_analyse_name)566    dist.parse_config_files()567    assert dist.get_version() == "42"568    assert set(dist.packages) == {"pkg"}569 570 571def _populate_project_dir(root, files, options):572    # NOTE: Currently pypa/build will refuse to build the project if no573    # `pyproject.toml` or `setup.py` is found. So it is impossible to do574    # completely "config-less" projects.575    basic = {576        "setup.py": "import setuptools\nsetuptools.setup()",577        "README.md": "# Example Package",578        "LICENSE": "Copyright (c) 2018",579    }580    jaraco.path.build(basic, prefix=root)581    _write_setupcfg(root, options)582    paths = (root / f for f in files)583    for path in paths:584        path.parent.mkdir(exist_ok=True, parents=True)585        path.touch()586 587 588def _write_setupcfg(root, options):589    if not options:590        print("~~~~~ **NO** setup.cfg ~~~~~")591        return592    setupcfg = ConfigParser()593    setupcfg.add_section("options")594    for key, value in options.items():595        if key == "packages.find":596            setupcfg.add_section(f"options.{key}")597            setupcfg[f"options.{key}"].update(value)598        elif isinstance(value, list):599            setupcfg["options"][key] = ", ".join(value)600        elif isinstance(value, dict):601            str_value = "\n".join(f"\t{k} = {v}" for k, v in value.items())602            setupcfg["options"][key] = "\n" + str_value603        else:604            setupcfg["options"][key] = str(value)605    with open(root / "setup.cfg", "w", encoding="utf-8") as f:606        setupcfg.write(f)607    print("~~~~~ setup.cfg ~~~~~")608    print((root / "setup.cfg").read_text(encoding="utf-8"))609 610 611def _run_build(path, *flags):612    cmd = [sys.executable, "-m", "build", "--no-isolation", *flags, str(path)]613    return run(cmd, env={'DISTUTILS_DEBUG': ''})614 615 616def _get_dist(dist_path, attrs):617    root = "/".join(os.path.split(dist_path))  # POSIX-style618 619    script = dist_path / 'setup.py'620    if script.exists():621        with Path(dist_path):622            dist = cast(623                Distribution,624                distutils.core.run_setup("setup.py", {}, stop_after="init"),625            )626    else:627        dist = Distribution(attrs)628 629    dist.src_root = root630    dist.script_name = "setup.py"631    with Path(dist_path):632        dist.parse_config_files()633 634    dist.set_defaults()635    return dist636 637 638def _run_sdist_programatically(dist_path, attrs):639    dist = _get_dist(dist_path, attrs)640    cmd = sdist(dist)641    cmd.ensure_finalized()642    assert cmd.distribution.packages or cmd.distribution.py_modules643 644    with quiet(), Path(dist_path):645        cmd.run()646 647    return dist, cmd648