Team Ai
Apppublic

openenv/echo_env

sourceHugging Faceupdated 1d agoView on Hugging Face
6likes
modal_provider.py698 linesDownload Raw Back to runtime
1# SPDX-License-Identifier: BSD-3-Clause2 3"""4Modal container provider for running OpenEnv environments in Modal sandboxes.5 6Requires the ``modal`` SDK: ``pip install modal>=1.4``7 8The provider boots an OpenEnv server inside a Modal sandbox, exposes it on an9encrypted tunnel, and returns an ``https://`` URL that ``EnvClient`` connects to10over ``wss://``.11 12Supports both the stable Sandbox API (``modal.Sandbox.create``) and the beta13Sandbox v2 API (``modal.Sandbox._experimental_create``). Sandbox v2 is opt-in14via ``use_sandbox_v2=True`` because it is an experimental, private SDK feature;15see https://modal.com/docs/guide/sandbox-v2 .16"""17 18from __future__ import annotations19 20import logging21import shlex22import time23from typing import Any, Dict, Optional24 25from ._server_config import parse_dockerfile_cmd, parse_openenv_app_field26from .providers import ContainerProvider27 28logger = logging.getLogger(__name__)29 30_DEFAULT_MODAL_PORT = 800031 32# Modal's default per-container resource requests. We pass configured cpu/memory33# as the *limit* half of Modal's ``(request, limit)`` tuple so they cap usage34# rather than reserving (and billing for) the full amount. See35# https://modal.com/docs/guide/resources .36_DEFAULT_CPU_REQUEST = 0.12537_DEFAULT_MEMORY_REQUEST = 12838 39 40def _require_secure_url(url: str) -> str:41    """Enforce https/wss transport (RFC 002 security invariant S1).42 43    ``EnvClient`` derives its WebSocket URL from this base URL, so a plaintext44    URL would become a cleartext ``ws://`` connection. The offending URL is45    deliberately omitted from the error because a Modal tunnel URL is a bearer46    capability that must not leak into logs.47    """48    if not isinstance(url, str) or not url.lower().startswith("https://"):49        raise RuntimeError(50            "Modal sandbox returned a non-HTTPS tunnel URL. OpenEnv requires an "51            "https/wss base_url so EnvClient traffic is encrypted. Refusing to "52            "connect over plaintext."53        )54    return url55 56 57class _DefaultModalAdapter:58    """Thin adapter over the ``modal`` SDK.59 60    The provider talks to this private adapter instead of spreading SDK details61    through its own logic; tests inject a duck-typed fake in its place. Keeping62    the SDK surface here also localizes the feature check for the private63    Sandbox v2 (``_experimental_create``) API.64    """65 66    def __init__(self, *, app_name: str, use_sandbox_v2: bool):67        import modal68 69        self._modal = modal70        self._app_name = app_name71        self._use_sandbox_v2 = use_sandbox_v272 73        # Sandbox v2 rides on a private SDK entry point that is not present in74        # every modal release. Fail fast at construction with clear guidance75        # rather than at start_container time with an AttributeError.76        if use_sandbox_v2 and not hasattr(modal.Sandbox, "_experimental_create"):77            raise RuntimeError(78                "use_sandbox_v2=True requires modal.Sandbox._experimental_create, "79                "which is not available in the installed modal SDK. Upgrade modal, "80                "or use the stable Sandbox API (use_sandbox_v2=False). Sandbox v2 "81                "is an experimental feature; contact support@modal.com for access."82            )83 84    def image_from_registry(self, tag: str) -> Any:85        return self._modal.Image.from_registry(tag)86 87    def image_from_dockerfile(self, dockerfile_path: str, context_dir: str) -> Any:88        return self._modal.Image.from_dockerfile(89            dockerfile_path, context_dir=context_dir90        )91 92    def create_sandbox(93        self,94        *,95        image: Any,96        encrypted_ports: list[int],97        timeout: int,98        env: Optional[dict[str, str]],99        extra: dict[str, Any],100    ) -> Any:101        app = self._modal.App.lookup(self._app_name, create_if_missing=True)102        kwargs: Dict[str, Any] = {103            "app": app,104            "image": image,105            "encrypted_ports": encrypted_ports,106            "timeout": timeout,107            **extra,108        }109        if env:110            kwargs["env"] = dict(env)111 112        # The sandbox is started with a keep-alive entrypoint; the server113        # command is launched via exec afterwards (see ModalProvider).114        if self._use_sandbox_v2:115            return self._modal.Sandbox._experimental_create(116                "sleep", "infinity", **kwargs117            )118        return self._modal.Sandbox.create("sleep", "infinity", **kwargs)119 120    def exec(self, sandbox: Any, command: str, *, timeout: int = 10) -> str:121        """Run *command* through a shell inside *sandbox* and return its stdout."""122        proc = sandbox.exec("bash", "-c", command, timeout=timeout)123        try:124            out = proc.stdout.read()125        except Exception:126            out = ""127        proc.wait()128        return out or ""129 130    def tunnel_url(self, sandbox: Any, port: int) -> str:131        return sandbox.tunnels()[port].url132 133    def terminate(self, sandbox: Any) -> None:134        sandbox.terminate()135 136 137class ModalProvider(ContainerProvider):138    """139    Container provider that runs environments in Modal sandboxes.140 141    ``start_container``'s ``image`` is either a registry tag142    (``"echo-env:latest"``) or a ``"dockerfile:<path>"`` reference returned by143    :meth:`image_from_dockerfile`. The server is exposed on an encrypted Modal144    tunnel and the returned ``https://`` URL is what ``EnvClient`` connects to145    over ``wss://``.146 147    The environment runs untrusted code, so the provider is secure by default:148    it enforces https/wss transport, treats the tunnel URL as a bearer secret149    (never interpolated into errors), and never surfaces raw sandbox output150    unless ``surface_server_logs=True``.151 152    Only one sandbox is active per provider: calling ``start_container`` again153    before ``stop_container()``/``close()`` raises ``RuntimeError`` rather than154    orphaning the running sandbox. ``close()`` (and context-manager exit) stops155    the active sandbox.156 157    Example:158        ```python159        with ModalProvider(app_name="openenv", cpu=2.0, memory=4096) as provider:160            image = ModalProvider.image_from_dockerfile(161                "envs/echo_env/server/Dockerfile"162            )163            base_url = provider.start_container(image)164            provider.wait_for_ready(base_url)165        # sandbox terminated on exit166        ```167 168    Sandbox v2 (beta) is opt-in:169        ```python170        provider = ModalProvider(app_name="openenv", use_sandbox_v2=True)171        ```172    """173 174    _dockerfile_registry: Dict[str, Dict[str, Any]] = {}175 176    def __init__(177        self,178        *,179        image: str | None = None,180        env_vars: dict[str, str] | None = None,181        app_name: str = "openenv",182        use_sandbox_v2: bool = False,183        timeout: int = 300,184        cmd: str | None = None,185        cpu: float | None = None,186        memory: int | None = None,187        surface_server_logs: bool = False,188        _adapter: Any = None,189    ):190        """191        Args:192            image (`str`, *optional*):193                Registry image tag or ``"dockerfile:<path>"`` source to use194                when ``start_container()`` is called without an image.195            env_vars (`dict`, *optional*):196                Environment variables to use when ``start_container()`` is197                called without explicit ``env_vars``.198            app_name (`str`, *optional*, defaults to `"openenv"`):199                Modal app name the sandbox is created under. Looked up (and200                created if missing) via ``modal.App.lookup``.201            use_sandbox_v2 (`bool`, *optional*, defaults to `False`):202                When `True`, sandboxes are created via the beta203                ``modal.Sandbox._experimental_create`` API (Sandbox v2). This is204                an experimental, private SDK feature and is therefore off by205                default; a feature check fails fast at construction if the206                installed SDK lacks it. See https://modal.com/docs/guide/sandbox-v2 .207            timeout (`int`, *optional*, defaults to `300`):208                Maximum sandbox lifetime in seconds.209            cmd (`str`, *optional*):210                Shell command to start the server inside the sandbox. When211                omitted, the command is auto-discovered from ``openenv.yaml``212                (falling back to the Dockerfile ``CMD``).213            cpu (`float`, *optional*):214                Hard CPU-core limit for the sandbox. Passed as the limit half of215                Modal's ``(request, limit)`` tuple (request stays at Modal's216                default), so it caps usage rather than reserving cores. When217                `None`, Modal's default applies.218            memory (`int`, *optional*):219                Hard memory limit in MiB for the sandbox (containers exceeding it220                are OOM-killed). Passed as the limit half of Modal's221                ``(request, limit)`` tuple. When `None`, Modal's default applies.222            surface_server_logs (`bool`, *optional*, defaults to `False`):223                When `False` (default), captured sandbox output is withheld from224                raised errors so secrets the workload printed cannot leak into225                orchestrator/CI logs. When `True`, a best-effort redacted,226                length-bounded excerpt is included in startup-crash errors.227        """228        self._image = image229        self._env_vars = env_vars230        self._app_name = app_name231        self._use_sandbox_v2 = use_sandbox_v2232        self._timeout = timeout233        self._cmd = cmd234        self._cpu = cpu235        self._memory = memory236        self._surface_server_logs = surface_server_logs237        self._sandbox: Any = None238        self._base_url: str | None = None239        # Injected env-var values, used to scrub captured server output before240        # it is ever surfaced in an error.241        self._redact_values: set[str] = set()242 243        if _adapter is None:244            # Import eagerly (inside the adapter) so SDK/configuration errors —245            # including the Sandbox v2 feature check — surface at construction.246            self._adapter: Any = _DefaultModalAdapter(247                app_name=app_name, use_sandbox_v2=use_sandbox_v2248            )249        else:250            self._adapter = _adapter251 252        if use_sandbox_v2:253            logger.info(254                "Using Modal Sandbox v2 (experimental). This feature must be "255                "explicitly enabled. Contact support@modal.com to get access."256            )257 258    def _discover_server_cmd(self, port: int = _DEFAULT_MODAL_PORT) -> str:259        """Discover the server command from ``openenv.yaml`` inside the sandbox.260 261        Finds the file, reads the ``app`` field, and constructs a command262        of the form ``cd <env_root> && python -m uvicorn <app> --host 0.0.0.0 --port <port>``.263 264        Raises:265            ValueError: If ``openenv.yaml`` is not found or lacks an ``app`` field.266        """267        yaml_path = self._find_openenv_yaml()268        if yaml_path is None:269            raise ValueError(270                "Could not find openenv.yaml inside the sandbox. "271                "Pass an explicit cmd= to ModalProvider or start_container()."272            )273 274        content = self._adapter.exec(self._sandbox, f"cat {shlex.quote(yaml_path)}")275        app = self._parse_app_field(content)276        if app is None:277            raise ValueError(278                f"openenv.yaml at {yaml_path} does not contain an 'app' field. "279                "Pass an explicit cmd= to ModalProvider or start_container()."280            )281 282        # The directory containing openenv.yaml is the env root283        env_root = yaml_path.rsplit("/", 1)[0]284        return (285            f"cd {shlex.quote(env_root)} && "286            f"python -m uvicorn {shlex.quote(app)} --host 0.0.0.0 --port {port}"287        )288 289    def _find_openenv_yaml(self) -> str | None:290        """Locate ``openenv.yaml`` inside the sandbox.291 292        Tries the modern layout path ``/app/env/openenv.yaml`` first,293        then falls back to a ``find`` command for the old layout.294        """295        # Fast path: modern Dockerfile layout296        out = self._adapter.exec(297            self._sandbox, "test -f /app/env/openenv.yaml && echo found"298        )299        if "found" in (out or ""):300            return "/app/env/openenv.yaml"301 302        # Fallback: search for it (redirect stderr so error messages303        # like "No such file or directory" don't get mistaken for paths).304        path = self._adapter.exec(305            self._sandbox,306            "find /app -maxdepth 4 -name openenv.yaml -print -quit 2>/dev/null",307        ).strip()308        if path and path.startswith("/"):309            return path310 311        return None312 313    @staticmethod314    def _parse_app_field(yaml_content: str) -> str | None:315        """Extract the ``app`` value from raw openenv.yaml content.316 317        Uses PyYAML to handle comments, quotes, and nested keys correctly.318        """319        return parse_openenv_app_field(yaml_content)320 321    @staticmethod322    def _parse_dockerfile_cmd(dockerfile_content: str) -> str | None:323        """Extract the server command from the last ``CMD`` in a Dockerfile.324 325        Handles exec form (``CMD ["prog", "arg"]``) and shell form326        (``CMD prog arg``).  When a Dockerfile has multiple ``CMD``327        instructions (e.g. multi-stage builds), the last one wins - same328        semantics as Docker itself.  Lines where ``CMD`` appears inside a329        comment are ignored.330 331        Returns:332            The command as a single string, or ``None`` if no ``CMD`` found.333        """334        return parse_dockerfile_cmd(dockerfile_content)335 336    @classmethod337    def image_from_dockerfile(338        cls,339        dockerfile_path: str,340        context_dir: str | None = None,341    ) -> str:342        """Validate a Dockerfile and return a ``dockerfile:`` URI for343        :meth:`start_container`.344 345        Eagerly validates the Dockerfile (existence, COPY sources) and stores346        its content in an internal registry.  The actual ``modal.Image`` is347        created later inside ``start_container``.348 349        Args:350            dockerfile_path (`str`):351                Path to the Dockerfile on disk.352            context_dir (`str`, *optional*):353                Build context directory.  Defaults to the Dockerfile's354                grandparent directory, matching the ``openenv init``355                convention where Dockerfiles live in356                ``<env>/server/Dockerfile`` and the build context is357                ``<env>/``.  Pass explicitly for non-standard layouts358                (e.g. ``context_dir="."`` for repo-root contexts).359 360        Returns:361            `str`: A ``"dockerfile:<abs_path>"`` string to pass to362            ``start_container``.363 364        Raises:365            FileNotFoundError: If *dockerfile_path* does not exist.366            ValueError: If *context_dir* is given but does not exist,367                or if COPY sources in the Dockerfile cannot be found368                under the resolved context directory.369        """370        import pathlib371        import re372 373        src = pathlib.Path(dockerfile_path).resolve()374        if not src.is_file():375            raise FileNotFoundError(f"Dockerfile not found: {dockerfile_path}")376 377        if context_dir is not None:378            ctx = pathlib.Path(context_dir)379            if not ctx.is_dir():380                raise ValueError(f"context_dir does not exist: {context_dir}")381        else:382            # Default: grandparent of the Dockerfile, matching the383            # openenv init layout (<env>/server/Dockerfile -> <env>/).384            ctx = src.parent.parent385 386        content = src.read_text()387 388        # Validate that COPY sources exist under the context directory.389        # This catches mismatches early (e.g. a Dockerfile expecting repo390        # root as context when we defaulted to the env directory).391        for line in content.splitlines():392            m = re.match(r"^\s*COPY\s+(?!--from=)(\S+)\s+", line, re.IGNORECASE)393            if not m:394                continue395            copy_src = m.group(1)396            if copy_src.startswith("/"):397                continue398            resolved = ctx / copy_src399            if not resolved.exists() and not any(ctx.glob(copy_src)):400                raise ValueError(401                    f"Dockerfile COPY source '{copy_src}' not found "402                    f"under context_dir '{ctx}'. This Dockerfile may "403                    f"expect a different build context (e.g. the repo "404                    f"root). Pass context_dir explicitly."405                )406 407        # Parse CMD from the Dockerfile so start_container can use it as a408        # fallback when openenv.yaml is unavailable.409        parsed_cmd = cls._parse_dockerfile_cmd(content)410 411        cls._dockerfile_registry[str(src)] = {412            "dockerfile_path": str(src),413            "context_dir": str(ctx),414            "server_cmd": parsed_cmd,415        }416 417        return f"dockerfile:{src}"418 419    def _build_image(self, image: str) -> Any:420        """Build the ``modal.Image`` for *image* (registry tag or dockerfile:)."""421        if image.startswith("dockerfile:"):422            dockerfile_path = image[len("dockerfile:") :]423            meta = self._dockerfile_registry.get(dockerfile_path)424            if meta is None:425                raise ValueError(426                    f"No registered Dockerfile metadata for {dockerfile_path}. "427                    "Call ModalProvider.image_from_dockerfile() first."428                )429            return self._adapter.image_from_dockerfile(430                meta["dockerfile_path"], meta["context_dir"]431            )432 433        # Plain registry tag (e.g. "echo-env:latest").434        return self._adapter.image_from_registry(image)435 436    def start_container(437        self,438        image: str | None = None,439        port: int | None = None,440        env_vars: dict[str, str] | None = None,441        **kwargs: Any,442    ) -> str:443        """444        Create a Modal sandbox from a Docker image or Dockerfile.445 446        The sandbox is started with a keep-alive process and the server447        command is launched via ``exec`` afterwards, mirroring the discovery448        flow used by other cloud providers. The server command is resolved in449        order:450 451        1. Explicit ``cmd`` passed to the constructor.452        2. ``cmd`` key in ``**kwargs`` (popped before forwarding).453        3. Auto-discovered from ``openenv.yaml`` inside the sandbox.454        4. ``CMD`` parsed from the Dockerfile (when *image* came from455           ``image_from_dockerfile``).456 457        Args:458            image (`str`, *optional*):459                Registry image tag (e.g. ``"echo-env:latest"``) or460                ``"dockerfile:<path>"`` returned by461                :meth:`image_from_dockerfile`. May be omitted when supplied to462                the constructor.463            port (`int`, *optional*):464                Must be ``None`` or ``8000``. Modal exposes port 8000 via an465                encrypted tunnel; other ports raise ``ValueError``.466            env_vars (`dict`, *optional*):467                Environment variables forwarded to the sandbox.468            **kwargs:469                ``cmd`` (`str`) to override the server command; any remaining470                keyword arguments are forwarded to ``modal.Sandbox.create``.471 472        Returns:473            `str`: HTTPS tunnel URL for the sandbox (base_url).474        """475        if self._sandbox is not None:476            raise RuntimeError(477                "ModalProvider already has an active sandbox. Call "478                "stop_container() (or close()) before starting another — a "479                "second start would orphan the running sandbox."480            )481 482        if port is not None and port != _DEFAULT_MODAL_PORT:483            raise ValueError(484                f"ModalProvider only supports port {_DEFAULT_MODAL_PORT} "485                f"(got {port}). The Modal tunnel routes to port "486                f"{_DEFAULT_MODAL_PORT} inside the sandbox."487            )488 489        effective_image = image if image is not None else self._image490        if effective_image is None:491            raise ValueError(492                "ModalProvider requires an image. Pass it to the constructor "493                "or start_container()."494            )495        effective_env_vars = self._env_vars if env_vars is None else env_vars496 497        # Resolve the server command (may be None; discovery happens after498        # sandbox creation when we can inspect the filesystem).499        cmd = kwargs.pop("cmd", None) or self._cmd500 501        # CMD parsed from Dockerfile (populated for "dockerfile:" images).502        parsed_cmd: str | None = None503        if effective_image.startswith("dockerfile:"):504            meta = self._dockerfile_registry.get(effective_image[len("dockerfile:") :])505            if meta is not None:506                parsed_cmd = meta.get("server_cmd")507 508        modal_image = self._build_image(effective_image)509 510        extra: Dict[str, Any] = dict(kwargs)511        if self._cpu is not None:512            extra["cpu"] = (_DEFAULT_CPU_REQUEST, self._cpu)513        if self._memory is not None:514            extra["memory"] = (_DEFAULT_MEMORY_REQUEST, self._memory)515 516        # Record injected secret values so captured server output can be517        # scrubbed before it is ever surfaced in an error.518        self._redact_values = {519            value for value in (effective_env_vars or {}).values() if value520        }521 522        # A create failure created nothing, so just drop the recorded secrets523        # and re-raise (the double-start guard above guarantees there is no524        # pre-existing sandbox to delete).525        try:526            self._sandbox = self._adapter.create_sandbox(527                image=modal_image,528                encrypted_ports=[_DEFAULT_MODAL_PORT],529                timeout=self._timeout,530                env=effective_env_vars,531                extra=extra,532            )533        except Exception:534            self._redact_values = set()535            raise536 537        try:538            # Discover server command from openenv.yaml if not explicitly set.539            if cmd is None:540                try:541                    cmd = self._discover_server_cmd()542                except ValueError:543                    # Fall back to CMD parsed from Dockerfile (if available).544                    if parsed_cmd:545                        cmd = parsed_cmd546                    else:547                        raise548 549            # Launch the server in the background. Write the PID so we can550            # check whether the process crashed in wait_for_ready().551            escaped_cmd = shlex.quote(cmd)552            self._adapter.exec(553                self._sandbox,554                f"nohup bash -c {escaped_cmd} > /tmp/openenv-server.log 2>&1 &"555                " echo $! > /tmp/openenv-server.pid",556            )557 558            # Resolve the public tunnel URL for port 8000.559            self._base_url = _require_secure_url(560                self._adapter.tunnel_url(self._sandbox, _DEFAULT_MODAL_PORT)561            )562        except Exception:563            # A cleanup failure here must not mask the original error: swallow564            # any exception from stop_container() so the root cause propagates.565            try:566                self.stop_container()567            except Exception:568                pass569            raise570 571        return self._base_url572 573    def stop_container(self) -> None:574        """Terminate the Modal sandbox."""575        if self._sandbox is None:576            # Still drop any injected secret values recorded by a failed start.577            self._redact_values = set()578            return579 580        try:581            self._adapter.terminate(self._sandbox)582        finally:583            self._sandbox = None584            self._base_url = None585            self._redact_values = set()586 587    def close(self) -> None:588        """Stop the active sandbox.589 590        Overrides the base no-op so a caller holding a bare ``ContainerProvider``591        reference can release the sandbox polymorphically (also invoked on592        context-manager exit). ``ModalProvider`` holds no separate SDK client, so593        this is equivalent to ``stop_container()``.594        """595        self.stop_container()596 597    @property598    def base_url(self) -> str:599        """URL returned by the last ``start_container``."""600        if self._base_url is None:601            raise RuntimeError(602                "ModalProvider has no active base_url. Start the provider "603                "before reading base_url."604            )605        return self._base_url606 607    def _redact(self, text: str, *, max_chars: int = 2000) -> str:608        """Scrub injected secret values and bound length before surfacing output.609 610        Replaces any injected env-var value with `***` and keeps only the tail.611        This is best-effort (exact-match only), which is why server output is612        withheld entirely unless ``surface_server_logs=True``.613        """614        redacted = text or ""615        for value in self._redact_values:616            redacted = redacted.replace(value, "***")617        if len(redacted) > max_chars:618            redacted = "...(truncated)...\n" + redacted[-max_chars:]619        return redacted620 621    def _server_died_message(self) -> str:622        """Build the startup-crash error, secure by default.623 624        Untrusted code can print secrets then force a crash to exfiltrate them625        through the exception (which lands in orchestrator/CI logs), so sandbox626        output is excluded unless ``surface_server_logs=True``, in which case a627        best-effort redacted, bounded excerpt is included.628        """629        base = (630            "Modal sandbox server process died during startup. Server output is "631            "not surfaced to avoid leaking secrets injected into the sandbox; "632            "retrieve /tmp/openenv-server.log from the sandbox out of band, or "633            "construct the provider with surface_server_logs=True to include a "634            "redacted excerpt."635        )636        if not self._surface_server_logs or self._sandbox is None:637            return base638 639        log = self._redact(640            self._adapter.exec(self._sandbox, "cat /tmp/openenv-server.log 2>/dev/null")641        )642        return (643            "Modal sandbox server process died during startup. The excerpt below "644            "is the sandbox server output with injected secret values redacted "645            "(best-effort); it may still contain secrets the workload printed "646            f"by other means.\nLog (redacted):\n{log}"647        )648 649    def wait_for_ready(self, base_url: str, timeout_s: float = 120.0) -> None:650        """651        Poll the /health endpoint until the sandbox is ready.652 653        Uses a longer default timeout (120s) than local Docker providers654        because Modal sandboxes may have cold-start latency.655 656        Args:657            base_url (`str`):658                Tunnel URL returned by ``start_container()``.659            timeout_s (`float`, *optional*, defaults to `120.0`):660                Maximum seconds to wait.661 662        Raises:663            TimeoutError: If the sandbox doesn't become ready in time.664            RuntimeError: If the server process died (detected via PID check).665        """666        import requests667 668        health_url = f"{base_url}/health"669 670        deadline = time.time() + timeout_s671        while time.time() < deadline:672            try:673                response = requests.get(health_url, timeout=5.0)674                if response.status_code == 200:675                    return676            except requests.RequestException:677                pass678 679            # Early exit: if the server process died, raise immediately680            # instead of waiting for the full health-check timeout.681            if self._sandbox is not None:682                out = self._adapter.exec(683                    self._sandbox,684                    "kill -0 $(cat /tmp/openenv-server.pid) 2>/dev/null"685                    " && echo RUNNING || echo DEAD",686                )687                if "DEAD" in (out or ""):688                    raise RuntimeError(self._server_died_message())689 690            time.sleep(1.0)691 692        # The tunnel URL is a bearer capability, so it is deliberately omitted693        # from the timeout error.694        raise TimeoutError(f"Modal sandbox did not become ready within {timeout_s}s.")695 696 697__all__ = ["ModalProvider"]698