openenv/coding_env

1# Copyright (c) Meta Platforms, Inc. and affiliates.2# All rights reserved.3#4# This source code is licensed under the BSD-style license found in the5# LICENSE file in the root directory of this source tree.6 7"""8Daytona container provider for running OpenEnv environments in Daytona cloud sandboxes.9 10Requires the ``daytona`` SDK: ``pip install daytona>=0.10``11"""12 13from __future__ import annotations14 15import json16import os17import shlex18import time19from typing import Any, Callable, Dict, Optional20 21import yaml22 23from .providers import ContainerProvider24 25 26class DaytonaProvider(ContainerProvider):27    """28    Container provider that runs environments in Daytona cloud sandboxes.29 30    Example:31        >>> provider = DaytonaProvider(api_key="your-key")32        >>> image = DaytonaProvider.image_from_dockerfile("envs/echo_env/server/Dockerfile")33        >>> base_url = provider.start_container(image)34        >>> provider.wait_for_ready(base_url)35        >>> provider.stop_container()36    """37 38    _dockerfile_registry: Dict[str, Dict[str, Any]] = {}39 40    def __init__(41        self,42        *,43        api_key: Optional[str] = None,44        public: bool = False,45        resources: Optional[Any] = None,46        auto_stop_interval: int = 15,47        target: Optional[str] = None,48        on_snapshot_create_logs: Optional[Callable[[str], None]] = None,49        cmd: Optional[str] = None,50        create_timeout: float = 300,51    ):52        """53        Args:54            api_key: Daytona API key. Falls back to ``DAYTONA_API_KEY`` env var.55            public: If True, the sandbox preview is publicly accessible.56            resources: Optional ``daytona.Resources`` instance for CPU/memory.57            auto_stop_interval: Minutes of inactivity before auto-stop (0 disables).58            target: Daytona target region (e.g. "us").59            on_snapshot_create_logs: Callback for snapshot build log lines.60            cmd: Shell command to start the server inside the sandbox.61            create_timeout: Seconds to wait for sandbox creation (default 300).62                Heavy images (e.g. with Playwright/Chromium) may need more.63        """64        from daytona import Daytona, DaytonaConfig65 66        config_kwargs: Dict[str, Any] = {}67        resolved_key = api_key or os.environ.get("DAYTONA_API_KEY")68        if resolved_key:69            config_kwargs["api_key"] = resolved_key70        if target:71            config_kwargs["target"] = target72 73        self._daytona = Daytona(DaytonaConfig(**config_kwargs))74        self._public = public75        self._resources = resources76        self._auto_stop_interval = auto_stop_interval77        self._on_snapshot_create_logs = on_snapshot_create_logs78        self._cmd = cmd79        self._create_timeout = create_timeout80        self._sandbox: Any = None81        self._preview_url: Optional[str] = None82 83    def _discover_server_cmd(self, sandbox: Any, port: int = 8000) -> str:84        """Discover the server command from ``openenv.yaml`` inside *sandbox*.85 86        Finds the file, reads the ``app`` field, and constructs a command87        of the form ``cd <env_root> && python -m uvicorn <app> --host 0.0.0.0 --port <port>``.88 89        Raises:90            ValueError: If ``openenv.yaml`` is not found or lacks an ``app`` field.91        """92        yaml_path = self._find_openenv_yaml(sandbox)93        if yaml_path is None:94            raise ValueError(95                "Could not find openenv.yaml inside the sandbox. "96                "Pass an explicit cmd= to DaytonaProvider or start_container()."97            )98 99        cat_resp = sandbox.process.exec(f"cat {shlex.quote(yaml_path)}", timeout=10)100        content = cat_resp.result if hasattr(cat_resp, "result") else str(cat_resp)101        app = self._parse_app_field(content)102        if app is None:103            raise ValueError(104                f"openenv.yaml at {yaml_path} does not contain an 'app' field. "105                "Pass an explicit cmd= to DaytonaProvider or start_container()."106            )107 108        # The directory containing openenv.yaml is the env root109        env_root = yaml_path.rsplit("/", 1)[0]110        return (111            f"cd {shlex.quote(env_root)} && "112            f"python -m uvicorn {shlex.quote(app)} --host 0.0.0.0 --port {port}"113        )114 115    def _find_openenv_yaml(self, sandbox: Any) -> Optional[str]:116        """Locate ``openenv.yaml`` inside the sandbox.117 118        Tries the modern layout path ``/app/env/openenv.yaml`` first,119        then falls back to a ``find`` command for the old layout.120        """121        # Fast path: modern Dockerfile layout122        resp = sandbox.process.exec(123            "test -f /app/env/openenv.yaml && echo found", timeout=10124        )125        out = resp.result if hasattr(resp, "result") else str(resp)126        if "found" in (out or ""):127            return "/app/env/openenv.yaml"128 129        # Fallback: search for it (redirect stderr so error messages130        # like "No such file or directory" don't get mistaken for paths).131        resp = sandbox.process.exec(132            "find /app -maxdepth 4 -name openenv.yaml -print -quit 2>/dev/null",133            timeout=10,134        )135        path = (resp.result if hasattr(resp, "result") else str(resp) or "").strip()136        if path and path.startswith("/"):137            return path138 139        return None140 141    @staticmethod142    def _parse_app_field(yaml_content: str) -> Optional[str]:143        """Extract the ``app`` value from raw openenv.yaml content.144 145        Uses PyYAML to handle comments, quotes, and nested keys correctly.146        """147        try:148            data = yaml.safe_load(yaml_content) or {}149        except Exception:150            return None151 152        if not isinstance(data, dict):153            return None154 155        value = data.get("app")156        if isinstance(value, str):157            value = value.strip()158            return value if value else None159        return None160 161    @staticmethod162    def _parse_dockerfile_cmd(dockerfile_content: str) -> Optional[str]:163        """Extract the server command from the last ``CMD`` in a Dockerfile.164 165        Handles exec form (``CMD ["prog", "arg"]``) and shell form166        (``CMD prog arg``).  When a Dockerfile has multiple ``CMD``167        instructions (e.g. multi-stage builds), the last one wins - same168        semantics as Docker itself.  Lines where ``CMD`` appears inside a169        comment are ignored.170 171        Returns:172            The command as a single string, or ``None`` if no ``CMD`` found.173        """174        import re175 176        last_cmd: Optional[str] = None177        for line in dockerfile_content.splitlines():178            stripped = line.strip()179            # Skip comments180            if stripped.startswith("#"):181                continue182            match = re.match(r"CMD\s+(.+)", stripped, flags=re.IGNORECASE)183            if match:184                last_cmd = match.group(1).strip()185 186        if last_cmd is None:187            return None188 189        # Exec form: CMD ["executable", "param1", ...]190        if last_cmd.startswith("["):191            try:192                parts = json.loads(last_cmd)193                if isinstance(parts, list) and all(isinstance(p, str) for p in parts):194                    return " ".join(parts)195            except (json.JSONDecodeError, TypeError):196                pass197 198        # Shell form: CMD executable param1 ...199        return last_cmd if last_cmd else None200 201    @staticmethod202    def strip_buildkit_syntax(dockerfile_content: str) -> str:203        """Remove BuildKit ``--mount=...`` flags from ``RUN`` instructions.204 205        Handles single-line flags, multi-line continuations, and multiple206        ``--mount`` flags spread across continuation lines. Only leading207        ``--mount`` flags are removed (before the actual command starts).208 209        Daytona's ``Image.from_dockerfile`` does not support BuildKit210        ``--mount`` syntax.  This helper strips the flags so that standard211        Dockerfiles (like the ones generated by ``openenv build``) can212        be used directly.213        """214        import re215 216        def strip_leading_mounts(text: str) -> str:217            remaining = text218            while True:219                match = re.match(r"\s*--mount=\S+\s*", remaining)220                if not match:221                    return remaining222                remaining = remaining[match.end() :]223 224        lines = dockerfile_content.split("\n")225        result: list[str] = []226        in_run = False227        in_mount_prefix = False228 229        for line in lines:230            line_out = line231            run_start = False232            if re.match(r"\s*RUN(\s+|$)", line, flags=re.IGNORECASE):233                in_run = True234                in_mount_prefix = True235                run_start = True236 237            if in_run and in_mount_prefix:238                original_ends_with_slash = line_out.rstrip().endswith("\\")239                if run_start:240                    match = re.match(r"(\s*RUN\s+)(.*)$", line_out, flags=re.IGNORECASE)241                    if match:242                        run_prefix, remainder = match.group(1), match.group(2)243                    else:244                        run_prefix, remainder = line_out, ""245                    new_remainder = strip_leading_mounts(remainder)246                    line_out = run_prefix + new_remainder247                    content_for_check = new_remainder248                else:249                    new_remainder = strip_leading_mounts(line_out)250                    line_out = new_remainder251                    content_for_check = new_remainder252 253                if original_ends_with_slash and not line_out.rstrip().endswith("\\"):254                    line_out = line_out.rstrip() + " \\"255 256                if content_for_check.strip() not in ("", "\\"):257                    in_mount_prefix = False258 259            if in_run and not line_out.rstrip().endswith("\\"):260                in_run = False261                in_mount_prefix = False262 263            result.append(line_out)264 265        return "\n".join(result)266 267    @classmethod268    def image_from_dockerfile(269        cls,270        dockerfile_path: str,271        context_dir: str | None = None,272    ) -> str:273        """Validate a Dockerfile and return a ``dockerfile:`` URI for274        :meth:`start_container`.275 276        Eagerly validates the Dockerfile (existence, COPY sources,277        BuildKit stripping) and stores the processed content in an278        internal registry.  The actual ``daytona.Image`` is created279        later inside ``start_container``.280 281        Args:282            dockerfile_path: Path to the Dockerfile on disk.283            context_dir: Build context directory.  Defaults to the284                Dockerfile's grandparent directory, matching the285                ``openenv init`` convention where Dockerfiles live in286                ``<env>/server/Dockerfile`` and the build context is287                ``<env>/``.  Pass explicitly for non-standard layouts288                (e.g. ``context_dir="."`` for repo-root contexts).289 290        Returns:291            A ``"dockerfile:<abs_path>"`` string to pass to292            ``start_container``.293 294        Raises:295            FileNotFoundError: If *dockerfile_path* does not exist.296            ValueError: If *context_dir* is given but does not exist,297                or if COPY sources in the Dockerfile cannot be found298                under the resolved context directory.299        """300        import pathlib301        import re302 303        src = pathlib.Path(dockerfile_path).resolve()304        if not src.is_file():305            raise FileNotFoundError(f"Dockerfile not found: {dockerfile_path}")306 307        if context_dir is not None:308            ctx = pathlib.Path(context_dir)309            if not ctx.is_dir():310                raise ValueError(f"context_dir does not exist: {context_dir}")311        else:312            # Default: grandparent of the Dockerfile, matching the313            # openenv init layout (<env>/server/Dockerfile -> <env>/).314            ctx = src.parent.parent315 316        content = src.read_text()317        stripped = cls.strip_buildkit_syntax(content)318 319        # Validate that COPY sources exist under the context directory.320        # This catches mismatches early (e.g. a Dockerfile expecting repo321        # root as context when we defaulted to the env directory).322        for line in stripped.splitlines():323            m = re.match(r"^\s*COPY\s+(?!--from=)(\S+)\s+", line, re.IGNORECASE)324            if not m:325                continue326            copy_src = m.group(1)327            if copy_src.startswith("/"):328                continue329            resolved = ctx / copy_src330            if not resolved.exists() and not any(ctx.glob(copy_src)):331                raise ValueError(332                    f"Dockerfile COPY source '{copy_src}' not found "333                    f"under context_dir '{ctx}'. This Dockerfile may "334                    f"expect a different build context (e.g. the repo "335                    f"root). Pass context_dir explicitly."336                )337 338        # Parse CMD from the original Dockerfile so start_container can339        # use it as a fallback when openenv.yaml is unavailable.340        parsed_cmd = cls._parse_dockerfile_cmd(content)341 342        cls._dockerfile_registry[str(src)] = {343            "stripped_content": stripped,344            "context_dir": str(ctx),345            "server_cmd": parsed_cmd,346        }347 348        return f"dockerfile:{src}"349 350    def start_container(351        self,352        image: str,353        port: Optional[int] = None,354        env_vars: Optional[Dict[str, str]] = None,355        **kwargs: Any,356    ) -> str:357        """358        Create a Daytona sandbox from a Docker image or snapshot.359 360        Daytona does not execute the image's CMD (known bug — ENTRYPOINT361        runs, CMD does not).  The server command is resolved in order:362 363        1. Explicit ``cmd`` passed to the constructor.364        2. ``cmd`` key in ``**kwargs`` (popped before forwarding).365        3. Auto-discovered from ``openenv.yaml`` inside the sandbox.366        4. ``CMD`` parsed from the Dockerfile (when *image* came from367           ``image_from_dockerfile``).368 369        Args:370            image: Docker image name (e.g. ``"echo-env:latest"``),371                   ``"snapshot:<name>"`` to create from a pre-built snapshot,372                   or ``"dockerfile:<path>"`` returned by373                   :meth:`image_from_dockerfile`.374            port: Must be ``None`` or ``8000``. Daytona exposes port 8000375                  via its preview proxy; other ports raise ``ValueError``.376            env_vars: Environment variables forwarded to the sandbox.377            **kwargs: ``cmd`` (str) to override the server command;378                remaining kwargs passed through to ``Daytona.create()``.379 380        Returns:381            HTTPS preview URL for the sandbox (base_url).382        """383        if port is not None and port != 8000:384            raise ValueError(385                f"DaytonaProvider only supports port 8000 (got {port}). "386                "The Daytona preview proxy routes to port 8000 inside the sandbox."387            )388 389        # Resolve the server command (may be None; discovery happens after390        # sandbox creation when we can inspect the filesystem).391        cmd = kwargs.pop("cmd", None) or self._cmd392 393        # CMD parsed from Dockerfile (populated for "dockerfile:" images).394        parsed_cmd: Optional[str] = None395 396        # Build creation params397        create_kwargs: Dict[str, Any] = {}398        if env_vars:399            create_kwargs["env_vars"] = env_vars400        if self._public:401            create_kwargs["public"] = True402        if self._auto_stop_interval != 15:403            create_kwargs["auto_stop_interval"] = self._auto_stop_interval404 405        if image.startswith("snapshot:"):406            from daytona import CreateSandboxFromSnapshotParams407 408            snapshot_name = image[len("snapshot:") :]409            params = CreateSandboxFromSnapshotParams(410                snapshot=snapshot_name, **create_kwargs411            )412        elif image.startswith("dockerfile:"):413            from daytona import CreateSandboxFromImageParams, Image414 415            dockerfile_path = image[len("dockerfile:") :]416            meta = self._dockerfile_registry.get(dockerfile_path)417            if meta is None:418                raise ValueError(419                    f"No registered Dockerfile metadata for {dockerfile_path}. "420                    "Call DaytonaProvider.image_from_dockerfile() first."421                )422 423            parsed_cmd = meta.get("server_cmd")424 425            # Build the daytona Image from the pre-stripped content.426            import pathlib427            import uuid428 429            ctx = pathlib.Path(meta["context_dir"])430            tmp_name = f".daytona-{uuid.uuid4().hex[:8]}.dockerfile"431            tmp_path = ctx / tmp_name432            try:433                tmp_path.write_text(meta["stripped_content"])434                daytona_image = Image.from_dockerfile(str(tmp_path))435            finally:436                tmp_path.unlink(missing_ok=True)437 438            img_kwargs: Dict[str, Any] = {439                "image": daytona_image,440                **create_kwargs,441            }442            if self._resources is not None:443                img_kwargs["resources"] = self._resources444            params = CreateSandboxFromImageParams(**img_kwargs)445        else:446            from daytona import CreateSandboxFromImageParams447 448            img_kwargs = {"image": image, **create_kwargs}449            if self._resources is not None:450                img_kwargs["resources"] = self._resources451            params = CreateSandboxFromImageParams(**img_kwargs)452 453        # Create sandbox454        extra: Dict[str, Any] = dict(kwargs)455        if self._on_snapshot_create_logs is not None:456            extra["on_snapshot_create_logs"] = self._on_snapshot_create_logs457 458        self._sandbox = self._daytona.create(459            params, timeout=self._create_timeout, **extra460        )461 462        try:463            # Discover server command from openenv.yaml if not explicitly set.464            if cmd is None:465                try:466                    cmd = self._discover_server_cmd(self._sandbox)467                except ValueError:468                    # Fall back to CMD parsed from Dockerfile (if available).469                    if parsed_cmd:470                        cmd = parsed_cmd471                    else:472                        raise473 474            # Wrap in bash -c so compound commands (cd ... && uvicorn ...)475            # are handled correctly by nohup.  Write PID so we can check476            # if the process crashed later in wait_for_ready().477            escaped_cmd = shlex.quote(cmd)478            self._sandbox.process.exec(479                f"nohup bash -c {escaped_cmd} > /tmp/openenv-server.log 2>&1 &"480                " echo $! > /tmp/openenv-server.pid",481                timeout=10,482            )483 484            # Get a signed preview URL for port 8000.  The token is485            # embedded in the URL itself so no extra headers are needed.486            signed = self._sandbox.create_signed_preview_url(487                8000, expires_in_seconds=86400488            )489            self._preview_url = signed.url490        except Exception:491            self.stop_container()492            raise493 494        return self._preview_url495 496    def refresh_preview_url(self) -> str:497        """Get a fresh signed preview URL (valid for 24h).498 499        Daytona signed URLs expire after at most 24 hours.  Call this to500        get a new one for long-running sessions.  The returned URL points501        to the same sandbox — clients will need to reconnect using it.502        """503        if self._sandbox is None:504            raise RuntimeError("No active sandbox to refresh URL for.")505        signed = self._sandbox.create_signed_preview_url(8000, expires_in_seconds=86400)506        self._preview_url = signed.url507        return self._preview_url508 509    def stop_container(self) -> None:510        """Delete the Daytona sandbox."""511        if self._sandbox is None:512            return513 514        try:515            self._daytona.delete(self._sandbox)516        finally:517            self._sandbox = None518            self._preview_url = None519 520    def wait_for_ready(self, base_url: str, timeout_s: float = 120.0) -> None:521        """522        Poll the /health endpoint until the sandbox is ready.523 524        Uses a longer default timeout (120s) than Docker providers because525        Daytona sandboxes may have cold-start latency.526 527        Args:528            base_url: Preview URL returned by ``start_container()``.529            timeout_s: Maximum seconds to wait.530 531        Raises:532            TimeoutError: If the sandbox doesn't become ready in time.533            RuntimeError: If the server process died (detected via PID check).534        """535        import requests536 537        health_url = f"{base_url}/health"538 539        deadline = time.time() + timeout_s540        while time.time() < deadline:541            try:542                response = requests.get(health_url, timeout=5.0)543                if response.status_code == 200:544                    return545            except requests.RequestException:546                pass547 548            # Early exit: if the server process died, raise immediately549            # instead of waiting for the full health-check timeout.550            if self._sandbox is not None:551                resp = self._sandbox.process.exec(552                    "kill -0 $(cat /tmp/openenv-server.pid) 2>/dev/null"553                    " && echo RUNNING || echo DEAD",554                    timeout=10,555                )556                out = resp.result if hasattr(resp, "result") else str(resp)557                if "DEAD" in (out or ""):558                    log_resp = self._sandbox.process.exec(559                        "cat /tmp/openenv-server.log 2>/dev/null", timeout=10560                    )561                    log = (562                        log_resp.result563                        if hasattr(log_resp, "result")564                        else str(log_resp)565                    )566                    raise RuntimeError(f"Server process died.\nLog:\n{log}")567 568            time.sleep(1.0)569 570        raise TimeoutError(571            f"Daytona sandbox at {base_url} did not become ready within {timeout_s}s"572        )573