codekingpro/portable-devtools

1"""Automatic discovery of Python modules and packages (for inclusion in the2distribution) and other config values.3 4For the purposes of this module, the following nomenclature is used:5 6- "src-layout": a directory representing a Python project that contains a "src"7  folder. Everything under the "src" folder is meant to be included in the8  distribution when packaging the project. Example::9 10    .11    ├── tox.ini12    ├── pyproject.toml13    └── src/14        └── mypkg/15            ├── __init__.py16            ├── mymodule.py17            └── my_data_file.txt18 19- "flat-layout": a Python project that does not use "src-layout" but instead20  have a directory under the project root for each package::21 22    .23    ├── tox.ini24    ├── pyproject.toml25    └── mypkg/26        ├── __init__.py27        ├── mymodule.py28        └── my_data_file.txt29 30- "single-module": a project that contains a single Python script direct under31  the project root (no directory used)::32 33    .34    ├── tox.ini35    ├── pyproject.toml36    └── mymodule.py37 38"""39 40from __future__ import annotations41 42import itertools43import os44from collections.abc import Iterable, Iterator, Mapping45from fnmatch import fnmatchcase46from glob import glob47from pathlib import Path48from typing import TYPE_CHECKING, ClassVar49 50import _distutils_hack.override  # noqa: F40151 52from ._path import StrPath53 54from distutils import log55from distutils.util import convert_path56 57if TYPE_CHECKING:58    from setuptools import Distribution59 60chain_iter = itertools.chain.from_iterable61 62 63def _valid_name(path: StrPath) -> bool:64    # Ignore invalid names that cannot be imported directly65    return os.path.basename(path).isidentifier()66 67 68class _Filter:69    """70    Given a list of patterns, create a callable that will be true only if71    the input matches at least one of the patterns.72    """73 74    def __init__(self, *patterns: str) -> None:75        self._patterns = dict.fromkeys(patterns)76 77    def __call__(self, item: str) -> bool:78        return any(fnmatchcase(item, pat) for pat in self._patterns)79 80    def __contains__(self, item: str) -> bool:81        return item in self._patterns82 83 84class _Finder:85    """Base class that exposes functionality for module/package finders"""86 87    ALWAYS_EXCLUDE: ClassVar[tuple[str, ...]] = ()88    DEFAULT_EXCLUDE: ClassVar[tuple[str, ...]] = ()89 90    @classmethod91    def find(92        cls,93        where: StrPath = '.',94        exclude: Iterable[str] = (),95        include: Iterable[str] = ('*',),96    ) -> list[str]:97        """Return a list of all Python items (packages or modules, depending on98        the finder implementation) found within directory ``where``.99 100        ``where`` is the root directory which will be searched.101        It should be supplied as a "cross-platform" (i.e. URL-style) path;102        it will be converted to the appropriate local path syntax.103 104        ``exclude`` is a sequence of names to exclude; ``*`` can be used105        as a wildcard in the names.106        When finding packages, ``foo.*`` will exclude all subpackages of ``foo``107        (but not ``foo`` itself).108 109        ``include`` is a sequence of names to include.110        If it's specified, only the named items will be included.111        If it's not specified, all found items will be included.112        ``include`` can contain shell style wildcard patterns just like113        ``exclude``.114        """115 116        exclude = exclude or cls.DEFAULT_EXCLUDE117        return list(118            cls._find_iter(119                convert_path(str(where)),120                _Filter(*cls.ALWAYS_EXCLUDE, *exclude),121                _Filter(*include),122            )123        )124 125    @classmethod126    def _find_iter(127        cls, where: StrPath, exclude: _Filter, include: _Filter128    ) -> Iterator[str]:129        raise NotImplementedError130 131 132class PackageFinder(_Finder):133    """134    Generate a list of all Python packages found within a directory135    """136 137    ALWAYS_EXCLUDE = ("ez_setup", "*__pycache__")138 139    @classmethod140    def _find_iter(141        cls, where: StrPath, exclude: _Filter, include: _Filter142    ) -> Iterator[str]:143        """144        All the packages found in 'where' that pass the 'include' filter, but145        not the 'exclude' filter.146        """147        for root, dirs, files in os.walk(str(where), followlinks=True):148            # Copy dirs to iterate over it, then empty dirs.149            all_dirs = dirs[:]150            dirs[:] = []151 152            for dir in all_dirs:153                full_path = os.path.join(root, dir)154                rel_path = os.path.relpath(full_path, where)155                package = rel_path.replace(os.path.sep, '.')156 157                # Skip directory trees that are not valid packages158                if '.' in dir or not cls._looks_like_package(full_path, package):159                    continue160 161                # Should this package be included?162                if include(package) and not exclude(package):163                    yield package164 165                # Early pruning if there is nothing else to be scanned166                if f"{package}*" in exclude or f"{package}.*" in exclude:167                    continue168 169                # Keep searching subdirectories, as there may be more packages170                # down there, even if the parent was excluded.171                dirs.append(dir)172 173    @staticmethod174    def _looks_like_package(path: StrPath, _package_name: str) -> bool:175        """Does a directory look like a package?"""176        return os.path.isfile(os.path.join(path, '__init__.py'))177 178 179class PEP420PackageFinder(PackageFinder):180    @staticmethod181    def _looks_like_package(_path: StrPath, _package_name: str) -> bool:182        return True183 184 185class ModuleFinder(_Finder):186    """Find isolated Python modules.187    This function will **not** recurse subdirectories.188    """189 190    @classmethod191    def _find_iter(192        cls, where: StrPath, exclude: _Filter, include: _Filter193    ) -> Iterator[str]:194        for file in glob(os.path.join(where, "*.py")):195            module, _ext = os.path.splitext(os.path.basename(file))196 197            if not cls._looks_like_module(module):198                continue199 200            if include(module) and not exclude(module):201                yield module202 203    _looks_like_module = staticmethod(_valid_name)204 205 206# We have to be extra careful in the case of flat layout to not include files207# and directories not meant for distribution (e.g. tool-related)208 209 210class FlatLayoutPackageFinder(PEP420PackageFinder):211    _EXCLUDE = (212        "ci",213        "bin",214        "debian",215        "doc",216        "docs",217        "documentation",218        "manpages",219        "news",220        "newsfragments",221        "changelog",222        "test",223        "tests",224        "unit_test",225        "unit_tests",226        "example",227        "examples",228        "scripts",229        "tools",230        "util",231        "utils",232        "python",233        "build",234        "dist",235        "venv",236        "env",237        "requirements",238        # ---- Task runners / Build tools ----239        "tasks",  # invoke240        "fabfile",  # fabric241        "site_scons",  # SCons242        # ---- Other tools ----243        "benchmark",244        "benchmarks",245        "exercise",246        "exercises",247        "htmlcov",  # Coverage.py248        # ---- Hidden directories/Private packages ----249        "[._]*",250    )251 252    DEFAULT_EXCLUDE = tuple(chain_iter((p, f"{p}.*") for p in _EXCLUDE))253    """Reserved package names"""254 255    @staticmethod256    def _looks_like_package(_path: StrPath, package_name: str) -> bool:257        names = package_name.split('.')258        # Consider PEP 561259        root_pkg_is_valid = names[0].isidentifier() or names[0].endswith("-stubs")260        return root_pkg_is_valid and all(name.isidentifier() for name in names[1:])261 262 263class FlatLayoutModuleFinder(ModuleFinder):264    DEFAULT_EXCLUDE = (265        "setup",266        "conftest",267        "test",268        "tests",269        "example",270        "examples",271        "build",272        # ---- Task runners ----273        "toxfile",274        "noxfile",275        "pavement",276        "dodo",277        "tasks",278        "fabfile",279        # ---- Other tools ----280        "[Ss][Cc]onstruct",  # SCons281        "conanfile",  # Connan: C/C++ build tool282        "manage",  # Django283        "benchmark",284        "benchmarks",285        "exercise",286        "exercises",287        # ---- Hidden files/Private modules ----288        "[._]*",289    )290    """Reserved top-level module names"""291 292 293def _find_packages_within(root_pkg: str, pkg_dir: StrPath) -> list[str]:294    nested = PEP420PackageFinder.find(pkg_dir)295    return [root_pkg] + [".".join((root_pkg, n)) for n in nested]296 297 298class ConfigDiscovery:299    """Fill-in metadata and options that can be automatically derived300    (from other metadata/options, the file system or conventions)301    """302 303    def __init__(self, distribution: Distribution) -> None:304        self.dist = distribution305        self._called = False306        self._disabled = False307        self._skip_ext_modules = False308 309    def _disable(self):310        """Internal API to disable automatic discovery"""311        self._disabled = True312 313    def _ignore_ext_modules(self):314        """Internal API to disregard ext_modules.315 316        Normally auto-discovery would not be triggered if ``ext_modules`` are set317        (this is done for backward compatibility with existing packages relying on318        ``setup.py`` or ``setup.cfg``). However, ``setuptools`` can call this function319        to ignore given ``ext_modules`` and proceed with the auto-discovery if320        ``packages`` and ``py_modules`` are not given (e.g. when using pyproject.toml321        metadata).322        """323        self._skip_ext_modules = True324 325    @property326    def _root_dir(self) -> StrPath:327        # The best is to wait until `src_root` is set in dist, before using _root_dir.328        return self.dist.src_root or os.curdir329 330    @property331    def _package_dir(self) -> dict[str, str]:332        if self.dist.package_dir is None:333            return {}334        return self.dist.package_dir335 336    def __call__(337        self, force: bool = False, name: bool = True, ignore_ext_modules: bool = False338    ) -> None:339        """Automatically discover missing configuration fields340        and modifies the given ``distribution`` object in-place.341 342        Note that by default this will only have an effect the first time the343        ``ConfigDiscovery`` object is called.344 345        To repeatedly invoke automatic discovery (e.g. when the project346        directory changes), please use ``force=True`` (or create a new347        ``ConfigDiscovery`` instance).348        """349        if force is False and (self._called or self._disabled):350            # Avoid overhead of multiple calls351            return352 353        self._analyse_package_layout(ignore_ext_modules)354        if name:355            self.analyse_name()  # depends on ``packages`` and ``py_modules``356 357        self._called = True358 359    def _explicitly_specified(self, ignore_ext_modules: bool) -> bool:360        """``True`` if the user has specified some form of package/module listing"""361        ignore_ext_modules = ignore_ext_modules or self._skip_ext_modules362        ext_modules = not (self.dist.ext_modules is None or ignore_ext_modules)363        return (364            self.dist.packages is not None365            or self.dist.py_modules is not None366            or ext_modules367            or hasattr(self.dist, "configuration")368            and self.dist.configuration369            # ^ Some projects use numpy.distutils.misc_util.Configuration370        )371 372    def _analyse_package_layout(self, ignore_ext_modules: bool) -> bool:373        if self._explicitly_specified(ignore_ext_modules):374            # For backward compatibility, just try to find modules/packages375            # when nothing is given376            return True377 378        log.debug(379            "No `packages` or `py_modules` configuration, performing "380            "automatic discovery."381        )382 383        return (384            self._analyse_explicit_layout()385            or self._analyse_src_layout()386            # flat-layout is the trickiest for discovery so it should be last387            or self._analyse_flat_layout()388        )389 390    def _analyse_explicit_layout(self) -> bool:391        """The user can explicitly give a package layout via ``package_dir``"""392        package_dir = self._package_dir.copy()  # don't modify directly393        package_dir.pop("", None)  # This falls under the "src-layout" umbrella394        root_dir = self._root_dir395 396        if not package_dir:397            return False398 399        log.debug(f"`explicit-layout` detected -- analysing {package_dir}")400        pkgs = chain_iter(401            _find_packages_within(pkg, os.path.join(root_dir, parent_dir))402            for pkg, parent_dir in package_dir.items()403        )404        self.dist.packages = list(pkgs)405        log.debug(f"discovered packages -- {self.dist.packages}")406        return True407 408    def _analyse_src_layout(self) -> bool:409        """Try to find all packages or modules under the ``src`` directory410        (or anything pointed by ``package_dir[""]``).411 412        The "src-layout" is relatively safe for automatic discovery.413        We assume that everything within is meant to be included in the414        distribution.415 416        If ``package_dir[""]`` is not given, but the ``src`` directory exists,417        this function will set ``package_dir[""] = "src"``.418        """419        package_dir = self._package_dir420        src_dir = os.path.join(self._root_dir, package_dir.get("", "src"))421        if not os.path.isdir(src_dir):422            return False423 424        log.debug(f"`src-layout` detected -- analysing {src_dir}")425        package_dir.setdefault("", os.path.basename(src_dir))426        self.dist.package_dir = package_dir  # persist eventual modifications427        self.dist.packages = PEP420PackageFinder.find(src_dir)428        self.dist.py_modules = ModuleFinder.find(src_dir)429        log.debug(f"discovered packages -- {self.dist.packages}")430        log.debug(f"discovered py_modules -- {self.dist.py_modules}")431        return True432 433    def _analyse_flat_layout(self) -> bool:434        """Try to find all packages and modules under the project root.435 436        Since the ``flat-layout`` is more dangerous in terms of accidentally including437        extra files/directories, this function is more conservative and will raise an438        error if multiple packages or modules are found.439 440        This assumes that multi-package dists are uncommon and refuse to support that441        use case in order to be able to prevent unintended errors.442        """443        log.debug(f"`flat-layout` detected -- analysing {self._root_dir}")444        return self._analyse_flat_packages() or self._analyse_flat_modules()445 446    def _analyse_flat_packages(self) -> bool:447        self.dist.packages = FlatLayoutPackageFinder.find(self._root_dir)448        top_level = remove_nested_packages(remove_stubs(self.dist.packages))449        log.debug(f"discovered packages -- {self.dist.packages}")450        self._ensure_no_accidental_inclusion(top_level, "packages")451        return bool(top_level)452 453    def _analyse_flat_modules(self) -> bool:454        self.dist.py_modules = FlatLayoutModuleFinder.find(self._root_dir)455        log.debug(f"discovered py_modules -- {self.dist.py_modules}")456        self._ensure_no_accidental_inclusion(self.dist.py_modules, "modules")457        return bool(self.dist.py_modules)458 459    def _ensure_no_accidental_inclusion(self, detected: list[str], kind: str):460        if len(detected) > 1:461            from inspect import cleandoc462 463            from setuptools.errors import PackageDiscoveryError464 465            msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}.466 467            To avoid accidental inclusion of unwanted files or directories,468            setuptools will not proceed with this build.469 470            If you are trying to create a single distribution with multiple {kind}471            on purpose, you should not rely on automatic discovery.472            Instead, consider the following options:473 474            1. set up custom discovery (`find` directive with `include` or `exclude`)475            2. use a `src-layout`476            3. explicitly set `py_modules` or `packages` with a list of names477 478            To find more information, look for "package discovery" on setuptools docs.479            """480            raise PackageDiscoveryError(cleandoc(msg))481 482    def analyse_name(self) -> None:483        """The packages/modules are the essential contribution of the author.484        Therefore the name of the distribution can be derived from them.485        """486        if self.dist.metadata.name or self.dist.name:487            # get_name() is not reliable (can return "UNKNOWN")488            return489 490        log.debug("No `name` configuration, performing automatic discovery")491 492        name = (493            self._find_name_single_package_or_module()494            or self._find_name_from_packages()495        )496        if name:497            self.dist.metadata.name = name498 499    def _find_name_single_package_or_module(self) -> str | None:500        """Exactly one module or package"""501        for field in ('packages', 'py_modules'):502            items = getattr(self.dist, field, None) or []503            if items and len(items) == 1:504                log.debug(f"Single module/package detected, name: {items[0]}")505                return items[0]506 507        return None508 509    def _find_name_from_packages(self) -> str | None:510        """Try to find the root package that is not a PEP 420 namespace"""511        if not self.dist.packages:512            return None513 514        packages = remove_stubs(sorted(self.dist.packages, key=len))515        package_dir = self.dist.package_dir or {}516 517        parent_pkg = find_parent_package(packages, package_dir, self._root_dir)518        if parent_pkg:519            log.debug(f"Common parent package detected, name: {parent_pkg}")520            return parent_pkg521 522        log.warn("No parent package detected, impossible to derive `name`")523        return None524 525 526def remove_nested_packages(packages: list[str]) -> list[str]:527    """Remove nested packages from a list of packages.528 529    >>> remove_nested_packages(["a", "a.b1", "a.b2", "a.b1.c1"])530    ['a']531    >>> remove_nested_packages(["a", "b", "c.d", "c.d.e.f", "g.h", "a.a1"])532    ['a', 'b', 'c.d', 'g.h']533    """534    pkgs = sorted(packages, key=len)535    top_level = pkgs[:]536    size = len(pkgs)537    for i, name in enumerate(reversed(pkgs)):538        if any(name.startswith(f"{other}.") for other in top_level):539            top_level.pop(size - i - 1)540 541    return top_level542 543 544def remove_stubs(packages: list[str]) -> list[str]:545    """Remove type stubs (:pep:`561`) from a list of packages.546 547    >>> remove_stubs(["a", "a.b", "a-stubs", "a-stubs.b.c", "b", "c-stubs"])548    ['a', 'a.b', 'b']549    """550    return [pkg for pkg in packages if not pkg.split(".")[0].endswith("-stubs")]551 552 553def find_parent_package(554    packages: list[str], package_dir: Mapping[str, str], root_dir: StrPath555) -> str | None:556    """Find the parent package that is not a namespace."""557    packages = sorted(packages, key=len)558    common_ancestors = []559    for i, name in enumerate(packages):560        if not all(n.startswith(f"{name}.") for n in packages[i + 1 :]):561            # Since packages are sorted by length, this condition is able562            # to find a list of all common ancestors.563            # When there is divergence (e.g. multiple root packages)564            # the list will be empty565            break566        common_ancestors.append(name)567 568    for name in common_ancestors:569        pkg_path = find_package_path(name, package_dir, root_dir)570        init = os.path.join(pkg_path, "__init__.py")571        if os.path.isfile(init):572            return name573 574    return None575 576 577def find_package_path(578    name: str, package_dir: Mapping[str, str], root_dir: StrPath579) -> str:580    """Given a package name, return the path where it should be found on581    disk, considering the ``package_dir`` option.582 583    >>> path = find_package_path("my.pkg", {"": "root/is/nested"}, ".")584    >>> path.replace(os.sep, "/")585    './root/is/nested/my/pkg'586 587    >>> path = find_package_path("my.pkg", {"my": "root/is/nested"}, ".")588    >>> path.replace(os.sep, "/")589    './root/is/nested/pkg'590 591    >>> path = find_package_path("my.pkg", {"my.pkg": "root/is/nested"}, ".")592    >>> path.replace(os.sep, "/")593    './root/is/nested'594 595    >>> path = find_package_path("other.pkg", {"my.pkg": "root/is/nested"}, ".")596    >>> path.replace(os.sep, "/")597    './other/pkg'598    """599    parts = name.split(".")600    for i in range(len(parts), 0, -1):601        # Look backwards, the most specific package_dir first602        partial_name = ".".join(parts[:i])603        if partial_name in package_dir:604            parent = package_dir[partial_name]605            return os.path.join(root_dir, parent, *parts[i:])606 607    parent = package_dir.get("") or ""608    return os.path.join(root_dir, *parent.split("/"), *parts)609 610 611def construct_package_dir(packages: list[str], package_path: StrPath) -> dict[str, str]:612    parent_pkgs = remove_nested_packages(packages)613    prefix = Path(package_path).parts614    return {pkg: "/".join([*prefix, *pkg.split(".")]) for pkg in parent_pkgs}615