codekingpro/portable-devtools

1"""Build Environment used for isolation during sdist building"""2 3from __future__ import annotations4 5import logging6import os7import pathlib8import site9import sys10import textwrap11from collections import OrderedDict12from collections.abc import Iterable, Sequence13from contextlib import AbstractContextManager as ContextManager14from contextlib import nullcontext15from io import StringIO16from types import TracebackType17from typing import TYPE_CHECKING, Protocol, TypedDict18 19from pip._vendor.packaging.version import Version20 21from pip import __file__ as pip_location22from pip._internal.cli.spinners import open_rich_spinner, open_spinner23from pip._internal.exceptions import (24    BuildDependencyInstallError,25    DiagnosticPipError,26    InstallWheelBuildError,27    PipError,28)29from pip._internal.locations import get_platlib, get_purelib, get_scheme30from pip._internal.metadata import get_default_environment, get_environment31from pip._internal.utils.deprecation import deprecated32from pip._internal.utils.logging import VERBOSE, capture_logging33from pip._internal.utils.packaging import get_requirement34from pip._internal.utils.subprocess import call_subprocess35from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds36 37if TYPE_CHECKING:38    from pip._internal.cache import WheelCache39    from pip._internal.index.package_finder import PackageFinder40    from pip._internal.operations.build.build_tracker import BuildTracker41    from pip._internal.req.req_install import InstallRequirement42    from pip._internal.resolution.base import BaseResolver43 44    class ExtraEnviron(TypedDict, total=False):45        extra_environ: dict[str, str]46 47 48logger = logging.getLogger(__name__)49 50 51def _dedup(a: str, b: str) -> tuple[str] | tuple[str, str]:52    return (a, b) if a != b else (a,)53 54 55class _Prefix:56    def __init__(self, path: str) -> None:57        self.path = path58        self.setup = False59        scheme = get_scheme("", prefix=path)60        self.bin_dir = scheme.scripts61        self.lib_dirs = _dedup(scheme.purelib, scheme.platlib)62 63 64def get_runnable_pip() -> str:65    """Get a file to pass to a Python executable, to run the currently-running pip.66 67    This is used to run a pip subprocess, for installing requirements into the build68    environment.69    """70    source = pathlib.Path(pip_location).resolve().parent71 72    if not source.is_dir():73        # This would happen if someone is using pip from inside a zip file. In that74        # case, we can use that directly.75        return str(source)76 77    return os.fsdecode(source / "__pip-runner__.py")78 79 80def _get_system_sitepackages() -> set[str]:81    """Get system site packages82 83    Usually from site.getsitepackages,84    but fallback on `get_purelib()/get_platlib()` if unavailable85    (e.g. in a virtualenv created by virtualenv<20)86 87    Returns normalized set of strings.88    """89    if hasattr(site, "getsitepackages"):90        system_sites = site.getsitepackages()91    else:92        # virtualenv < 20 overwrites site.py without getsitepackages93        # fallback on get_purelib/get_platlib.94        # this is known to miss things, but shouldn't in the cases95        # where getsitepackages() has been removed (inside a virtualenv)96        system_sites = [get_purelib(), get_platlib()]97    return {os.path.normcase(path) for path in system_sites}98 99 100class BuildEnvironmentInstaller(Protocol):101    """102    Interface for installing build dependencies into an isolated build103    environment.104    """105 106    def install(107        self,108        requirements: Iterable[str],109        prefix: _Prefix,110        *,111        kind: str,112        for_req: InstallRequirement | None,113    ) -> None: ...114 115 116class SubprocessBuildEnvironmentInstaller:117    """118    Install build dependencies by calling pip in a subprocess.119    """120 121    def __init__(122        self,123        finder: PackageFinder,124        build_constraints: list[str] | None = None,125        build_constraint_feature_enabled: bool = False,126    ) -> None:127        self.finder = finder128        self._build_constraints = build_constraints or []129        self._build_constraint_feature_enabled = build_constraint_feature_enabled130 131    def _deprecation_constraint_check(self) -> None:132        """133        Check for deprecation warning: PIP_CONSTRAINT affecting build environments.134 135        This warns when build-constraint feature is NOT enabled and PIP_CONSTRAINT136        is not empty.137        """138        if self._build_constraint_feature_enabled or self._build_constraints:139            return140 141        pip_constraint = os.environ.get("PIP_CONSTRAINT")142        if not pip_constraint or not pip_constraint.strip():143            return144 145        deprecated(146            reason=(147                "Setting PIP_CONSTRAINT will not affect "148                "build constraints in the future,"149            ),150            replacement=(151                "to specify build constraints using --build-constraint or "152                "PIP_BUILD_CONSTRAINT. To disable this warning without "153                "any build constraints set --use-feature=build-constraint or "154                'PIP_USE_FEATURE="build-constraint"'155            ),156            gone_in="26.2",157            issue=None,158        )159 160    def install(161        self,162        requirements: Iterable[str],163        prefix: _Prefix,164        *,165        kind: str,166        for_req: InstallRequirement | None,167    ) -> None:168        self._deprecation_constraint_check()169 170        finder = self.finder171        args: list[str] = [172            sys.executable,173            get_runnable_pip(),174            "install",175            "--ignore-installed",176            "--no-user",177            "--prefix",178            prefix.path,179            "--no-warn-script-location",180            "--disable-pip-version-check",181            # As the build environment is ephemeral, it's wasteful to182            # pre-compile everything, especially as not every Python183            # module will be used/compiled in most cases.184            "--no-compile",185            # The prefix specified two lines above, thus186            # target from config file or env var should be ignored187            "--target",188            "",189        ]190        if logger.getEffectiveLevel() <= logging.DEBUG:191            args.append("-vv")192        elif logger.getEffectiveLevel() <= VERBOSE:193            args.append("-v")194        for format_control in ("no_binary", "only_binary"):195            formats = getattr(finder.format_control, format_control)196            args.extend(197                (198                    "--" + format_control.replace("_", "-"),199                    ",".join(sorted(formats or {":none:"})),200                )201            )202 203        if finder.release_control is not None:204            # Use ordered args to preserve the user's original command-line order205            # This is important because later flags can override earlier ones206            for attr_name, value in finder.release_control.get_ordered_args():207                args.extend(("--" + attr_name.replace("_", "-"), value))208 209        index_urls = finder.index_urls210        if index_urls:211            args.extend(["-i", index_urls[0]])212            for extra_index in index_urls[1:]:213                args.extend(["--extra-index-url", extra_index])214        else:215            args.append("--no-index")216        for link in finder.find_links:217            args.extend(["--find-links", link])218 219        if finder.proxy:220            args.extend(["--proxy", finder.proxy])221        for host in finder.trusted_hosts:222            args.extend(["--trusted-host", host])223        if finder.custom_cert:224            args.extend(["--cert", finder.custom_cert])225        if finder.client_cert:226            args.extend(["--client-cert", finder.client_cert])227        if finder.prefer_binary:228            args.append("--prefer-binary")229 230        # Handle build constraints231        if self._build_constraint_feature_enabled:232            args.extend(["--use-feature", "build-constraint"])233 234        if self._build_constraints:235            # Build constraints must be passed as both constraints236            # and build constraints, so that nested builds receive237            # build constraints238            for constraint_file in self._build_constraints:239                args.extend(["--constraint", constraint_file])240                args.extend(["--build-constraint", constraint_file])241 242        extra_environ: ExtraEnviron = {}243        if self._build_constraint_feature_enabled and not self._build_constraints:244            # If there are no build constraints but the build constraints245            # feature is enabled then we must ignore regular constraints246            # in the isolated build environment247            extra_environ = {"extra_environ": {"_PIP_IN_BUILD_IGNORE_CONSTRAINTS": "1"}}248 249        if finder.uploaded_prior_to:250            args.extend(["--uploaded-prior-to", finder.uploaded_prior_to.isoformat()])251        args.append("--")252        args.extend(requirements)253 254        identify_requirement = (255            f" for {for_req.name}" if for_req and for_req.name else ""256        )257        with open_spinner(f"Installing {kind}") as spinner:258            call_subprocess(259                args,260                command_desc=f"installing {kind}{identify_requirement}",261                spinner=spinner,262                **extra_environ,263            )264 265 266class InprocessBuildEnvironmentInstaller:267    """268    Build dependency installer that runs in the same pip process.269 270    This contains a stripped down version of the install command with271    only the logic necessary for installing build dependencies. The272    finder, session, build tracker, and wheel cache are reused, but new273    instances of everything else are created as needed.274 275    Options are inherited from the parent install command unless276    they don't make sense for build dependencies (in which case, they277    are hard-coded, see comments below).278    """279 280    def __init__(281        self,282        *,283        finder: PackageFinder,284        build_tracker: BuildTracker,285        wheel_cache: WheelCache,286        build_constraints: Sequence[InstallRequirement] = (),287        verbosity: int = 0,288    ) -> None:289        from pip._internal.operations.prepare import RequirementPreparer290 291        self._finder = finder292        self._build_constraints = build_constraints293        self._wheel_cache = wheel_cache294        self._level = 0295 296        build_dir = TempDirectory(kind="build-env-install", globally_managed=True)297        self._preparer = RequirementPreparer(298            build_isolation_installer=self,299            # Inherited options or state.300            finder=finder,301            session=finder._link_collector.session,302            build_dir=build_dir.path,303            build_tracker=build_tracker,304            verbosity=verbosity,305            # This is irrelevant as it only applies to editable requirements.306            src_dir="",307            # Hard-coded options (that should NOT be inherited).308            download_dir=None,309            build_isolation=True,310            check_build_deps=False,311            progress_bar="off",312            # TODO: hash-checking should be extended to build deps, but that is313            # deferred for later as it'd be a breaking change.314            require_hashes=False,315            use_user_site=False,316            lazy_wheel=False,317            legacy_resolver=False,318        )319 320    def install(321        self,322        requirements: Iterable[str],323        prefix: _Prefix,324        *,325        kind: str,326        for_req: InstallRequirement | None,327    ) -> None:328        """Install entrypoint. Manages output capturing and error handling."""329        capture_logs = not logger.isEnabledFor(VERBOSE) and self._level == 0330        if capture_logs:331            # Hide the logs from the installation of build dependencies.332            # They will be shown only if an error occurs.333            capture_ctx: ContextManager[StringIO] = capture_logging()334            spinner: ContextManager[None] = open_rich_spinner(f"Installing {kind}")335        else:336            # Otherwise, pass-through all logs (with a header).337            capture_ctx, spinner = nullcontext(StringIO()), nullcontext()338            logger.info("Installing %s ...", kind)339 340        try:341            self._level += 1342            with spinner, capture_ctx as stream:343                self._install_impl(requirements, prefix)344 345        except DiagnosticPipError as exc:346            # Format similar to a nested subprocess error, where the347            # causing error is shown first, followed by the build error.348            logger.info(textwrap.dedent(stream.getvalue()))349            logger.error("%s", exc, extra={"rich": True})350            logger.info("")351            raise BuildDependencyInstallError(352                for_req, requirements, cause=exc, log_lines=None353            )354 355        except Exception as exc:356            logs: list[str] | None = textwrap.dedent(stream.getvalue()).splitlines()357            if not capture_logs:358                # If logs aren't being captured, then display the error inline359                # with the rest of the logs.360                logs = None361                if isinstance(exc, PipError):362                    logger.error("%s", exc)363                else:364                    logger.exception("pip crashed unexpectedly")365            raise BuildDependencyInstallError(366                for_req, requirements, cause=exc, log_lines=logs367            )368 369        finally:370            self._level -= 1371 372    def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None:373        """Core build dependency install logic."""374        from pip._internal.commands.install import installed_packages_summary375        from pip._internal.req import install_given_reqs376        from pip._internal.req.constructors import install_req_from_line377        from pip._internal.wheel_builder import build378 379        ireqs = [install_req_from_line(req, user_supplied=True) for req in requirements]380        ireqs.extend(self._build_constraints)381 382        resolver = self._make_resolver()383        resolved_set = resolver.resolve(ireqs, check_supported_wheels=True)384        self._preparer.prepare_linked_requirements_more(385            resolved_set.requirements.values()386        )387 388        reqs_to_build = [389            r for r in resolved_set.requirements_to_install if not r.is_wheel390        ]391        _, build_failures = build(reqs_to_build, self._wheel_cache, verify=True)392        if build_failures:393            raise InstallWheelBuildError(build_failures)394 395        installed = install_given_reqs(396            resolver.get_installation_order(resolved_set),397            prefix=prefix.path,398            # Hard-coded options (that should NOT be inherited).399            root=None,400            home=None,401            warn_script_location=False,402            use_user_site=False,403            # As the build environment is ephemeral, it's wasteful to404            # pre-compile everything since not all modules will be used.405            pycompile=False,406            progress_bar="off",407        )408 409        env = get_environment(list(prefix.lib_dirs))410        if summary := installed_packages_summary(installed, env):411            logger.info(summary)412 413    def _make_resolver(self) -> BaseResolver:414        """Create a new resolver for one time use."""415        # Legacy installer never used the legacy resolver so create a416        # resolvelib resolver directly. Yuck.417        from pip._internal.req.constructors import install_req_from_req_string418        from pip._internal.resolution.resolvelib.resolver import Resolver419 420        return Resolver(421            make_install_req=install_req_from_req_string,422            # Inherited state.423            preparer=self._preparer,424            finder=self._finder,425            wheel_cache=self._wheel_cache,426            # Hard-coded options (that should NOT be inherited).427            ignore_requires_python=False,428            use_user_site=False,429            ignore_dependencies=False,430            ignore_installed=True,431            force_reinstall=False,432            upgrade_strategy="to-satisfy-only",433            py_version_info=None,434        )435 436 437class BuildEnvironment:438    """Creates and manages an isolated environment to install build deps"""439 440    def __init__(self, installer: BuildEnvironmentInstaller) -> None:441        self.installer = installer442        temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)443 444        self._prefixes = OrderedDict(445            (name, _Prefix(os.path.join(temp_dir.path, name)))446            for name in ("normal", "overlay")447        )448 449        self._bin_dirs: list[str] = []450        self._lib_dirs: list[str] = []451        for prefix in reversed(list(self._prefixes.values())):452            self._bin_dirs.append(prefix.bin_dir)453            self._lib_dirs.extend(prefix.lib_dirs)454 455        # Customize site to:456        # - ensure .pth files are honored457        # - prevent access to system site packages458        system_sites = _get_system_sitepackages()459 460        self._site_dir = os.path.join(temp_dir.path, "site")461        if not os.path.exists(self._site_dir):462            os.mkdir(self._site_dir)463        with open(464            os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"465        ) as fp:466            fp.write(467                textwrap.dedent(468                    """469                import os, site, sys470 471                # First, drop system-sites related paths.472                original_sys_path = sys.path[:]473                known_paths = set()474                for path in {system_sites!r}:475                    site.addsitedir(path, known_paths=known_paths)476                system_paths = set(477                    os.path.normcase(path)478                    for path in sys.path[len(original_sys_path):]479                )480                original_sys_path = [481                    path for path in original_sys_path482                    if os.path.normcase(path) not in system_paths483                ]484                sys.path = original_sys_path485 486                # Second, add lib directories.487                # ensuring .pth file are processed.488                for path in {lib_dirs!r}:489                    assert not path in sys.path490                    site.addsitedir(path)491                """492                ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)493            )494 495    def __enter__(self) -> None:496        self._save_env = {497            name: os.environ.get(name, None)498            for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")499        }500 501        path = self._bin_dirs[:]502        old_path = self._save_env["PATH"]503        if old_path:504            path.extend(old_path.split(os.pathsep))505 506        pythonpath = [self._site_dir]507 508        os.environ.update(509            {510                "PATH": os.pathsep.join(path),511                "PYTHONNOUSERSITE": "1",512                "PYTHONPATH": os.pathsep.join(pythonpath),513            }514        )515 516    def __exit__(517        self,518        exc_type: type[BaseException] | None,519        exc_val: BaseException | None,520        exc_tb: TracebackType | None,521    ) -> None:522        for varname, old_value in self._save_env.items():523            if old_value is None:524                os.environ.pop(varname, None)525            else:526                os.environ[varname] = old_value527 528    def check_requirements(529        self, reqs: Iterable[str]530    ) -> tuple[set[tuple[str, str]], set[str]]:531        """Return 2 sets:532        - conflicting requirements: set of (installed, wanted) reqs tuples533        - missing requirements: set of reqs534        """535        missing = set()536        conflicting = set()537        if reqs:538            env = (539                get_environment(self._lib_dirs)540                if hasattr(self, "_lib_dirs")541                else get_default_environment()542            )543            for req_str in reqs:544                req = get_requirement(req_str)545                # We're explicitly evaluating with an empty extra value, since build546                # environments are not provided any mechanism to select specific extras.547                if req.marker is not None and not req.marker.evaluate({"extra": ""}):548                    continue549                dist = env.get_distribution(req.name)550                if not dist:551                    missing.add(req_str)552                    continue553                if isinstance(dist.version, Version):554                    installed_req_str = f"{req.name}=={dist.version}"555                else:556                    installed_req_str = f"{req.name}==={dist.version}"557                if not req.specifier.contains(dist.version, prereleases=True):558                    conflicting.add((installed_req_str, req_str))559                # FIXME: Consider direct URL?560        return conflicting, missing561 562    def install_requirements(563        self,564        requirements: Iterable[str],565        prefix_as_string: str,566        *,567        kind: str,568        for_req: InstallRequirement | None = None,569    ) -> None:570        prefix = self._prefixes[prefix_as_string]571        assert not prefix.setup572        prefix.setup = True573        if not requirements:574            return575        self.installer.install(requirements, prefix, kind=kind, for_req=for_req)576 577 578class NoOpBuildEnvironment(BuildEnvironment):579    """A no-op drop-in replacement for BuildEnvironment"""580 581    def __init__(self) -> None:582        pass583 584    def __enter__(self) -> None:585        pass586 587    def __exit__(588        self,589        exc_type: type[BaseException] | None,590        exc_val: BaseException | None,591        exc_tb: TracebackType | None,592    ) -> None:593        pass594 595    def cleanup(self) -> None:596        pass597 598    def install_requirements(599        self,600        requirements: Iterable[str],601        prefix_as_string: str,602        *,603        kind: str,604        for_req: InstallRequirement | None = None,605    ) -> None:606        raise NotImplementedError()607