openenv/echo_env
6
1# SPDX-License-Identifier: BSD-3-Clause2 3"""4Validation utilities for multi-mode deployment readiness.5 6This module provides functions to check if environments are properly7configured for multi-mode deployment (Docker, direct Python, notebooks, clusters).8"""9 10import ast11import re12from pathlib import Path13from typing import Any14from urllib.parse import urlparse15 16import requests17 18try:19 import tomllib20except ModuleNotFoundError:21 import tomli as tomllib22 23 24def _make_criterion(25 criterion_id: str,26 description: str,27 passed: bool,28 *,29 required: bool = True,30 details: str | None = None,31 expected: Any | None = None,32 actual: Any | None = None,33) -> dict[str, Any]:34 """Create a standard criterion result payload."""35 criterion: dict[str, Any] = {36 "id": criterion_id,37 "description": description,38 "passed": passed,39 "required": required,40 }41 if details is not None:42 criterion["details"] = details43 if expected is not None:44 criterion["expected"] = expected45 if actual is not None:46 criterion["actual"] = actual47 return criterion48 49 50def _normalize_runtime_url(base_url: str) -> str:51 """Normalize and validate a runtime target URL."""52 target = base_url.strip()53 if not target:54 raise ValueError("Runtime URL cannot be empty")55 56 if "://" not in target:57 target = f"http://{target}"58 59 parsed = urlparse(target)60 if not parsed.scheme or not parsed.netloc:61 raise ValueError(f"Invalid runtime URL: {base_url}")62 63 return target.rstrip("/")64 65 66def _runtime_standard_profile(api_version: str) -> str:67 """Resolve the runtime standard profile for an API version."""68 if api_version.startswith("1."):69 return "openenv-http/1.x"70 return "openenv-http/unknown"71 72 73def _build_summary(criteria: list[dict[str, Any]]) -> dict[str, Any]:74 """Build a compact pass/fail summary for a criteria list."""75 total_count = len(criteria)76 passed_count = sum(1 for criterion in criteria if criterion.get("passed", False))77 failed_criteria = [78 criterion.get("id", "unknown")79 for criterion in criteria80 if not criterion.get("passed", False)81 ]82 required_criteria = [83 criterion for criterion in criteria if criterion.get("required", True)84 ]85 required_total_count = len(required_criteria)86 required_passed_count = sum(87 1 for criterion in required_criteria if criterion.get("passed", False)88 )89 90 return {91 "passed_count": passed_count,92 "total_count": total_count,93 "failed_criteria": failed_criteria,94 "required_passed_count": required_passed_count,95 "required_total_count": required_total_count,96 }97 98 99def validate_running_environment(100 base_url: str, timeout_s: float = 5.0101) -> dict[str, Any]:102 """103 Validate a running OpenEnv server against runtime API standards.104 105 The returned JSON report contains an overall pass/fail result and106 per-criterion outcomes that can be consumed in CI.107 """108 normalized_url = _normalize_runtime_url(base_url)109 criteria: list[dict[str, Any]] = []110 111 report: dict[str, Any] = {112 "target": normalized_url,113 "validation_type": "running_environment",114 "standard_version": "unknown",115 "standard_profile": "openenv-http/unknown",116 "mode": "unknown",117 "passed": False,118 "summary": {},119 "criteria": criteria,120 }121 122 openapi_paths: dict[str, Any] = {}123 api_version = "unknown"124 125 # Criterion: OpenAPI endpoint reachable with a declared version.126 try:127 openapi_response = requests.get(128 f"{normalized_url}/openapi.json", timeout=timeout_s129 )130 except requests.RequestException as exc:131 criteria.append(132 _make_criterion(133 "openapi_version_available",134 "GET /openapi.json returns OpenAPI info.version",135 False,136 details=f"Request failed: {type(exc).__name__}: {exc}",137 expected={"status_code": 200, "info.version": "string"},138 )139 )140 else:141 try:142 openapi_json = openapi_response.json()143 except ValueError:144 openapi_json = None145 146 openapi_ok = (147 openapi_response.status_code == 200148 and isinstance(openapi_json, dict)149 and isinstance(openapi_json.get("info"), dict)150 and isinstance(openapi_json["info"].get("version"), str)151 )152 153 if openapi_ok:154 api_version = str(openapi_json["info"]["version"])155 openapi_paths = openapi_json.get("paths", {})156 criteria.append(157 _make_criterion(158 "openapi_version_available",159 "GET /openapi.json returns OpenAPI info.version",160 True,161 expected={"status_code": 200, "info.version": "string"},162 actual={163 "status_code": openapi_response.status_code,164 "info.version": api_version,165 },166 )167 )168 else:169 criteria.append(170 _make_criterion(171 "openapi_version_available",172 "GET /openapi.json returns OpenAPI info.version",173 False,174 details="Response missing required OpenAPI info.version field",175 expected={"status_code": 200, "info.version": "string"},176 actual={177 "status_code": openapi_response.status_code,178 "body_type": (179 type(openapi_json).__name__180 if openapi_json is not None181 else "non_json"182 ),183 },184 )185 )186 187 report["standard_version"] = api_version188 report["standard_profile"] = _runtime_standard_profile(api_version)189 190 # Criterion: Health endpoint.191 try:192 health_response = requests.get(f"{normalized_url}/health", timeout=timeout_s)193 except requests.RequestException as exc:194 criteria.append(195 _make_criterion(196 "health_endpoint",197 "GET /health returns healthy status",198 False,199 details=f"Request failed: {type(exc).__name__}: {exc}",200 expected={"status_code": 200, "status": "healthy"},201 )202 )203 else:204 try:205 health_json = health_response.json()206 except ValueError:207 health_json = None208 209 health_ok = (210 health_response.status_code == 200211 and isinstance(health_json, dict)212 and health_json.get("status") == "healthy"213 )214 criteria.append(215 _make_criterion(216 "health_endpoint",217 "GET /health returns healthy status",218 health_ok,219 expected={"status_code": 200, "status": "healthy"},220 actual={221 "status_code": health_response.status_code,222 "status": (223 health_json.get("status")224 if isinstance(health_json, dict)225 else None226 ),227 },228 )229 )230 231 # Criterion: Metadata endpoint has required fields.232 try:233 metadata_response = requests.get(234 f"{normalized_url}/metadata", timeout=timeout_s235 )236 except requests.RequestException as exc:237 criteria.append(238 _make_criterion(239 "metadata_endpoint",240 "GET /metadata returns name and description",241 False,242 details=f"Request failed: {type(exc).__name__}: {exc}",243 expected={"status_code": 200, "fields": ["name", "description"]},244 )245 )246 else:247 try:248 metadata_json = metadata_response.json()249 except ValueError:250 metadata_json = None251 252 metadata_ok = (253 metadata_response.status_code == 200254 and isinstance(metadata_json, dict)255 and isinstance(metadata_json.get("name"), str)256 and isinstance(metadata_json.get("description"), str)257 )258 criteria.append(259 _make_criterion(260 "metadata_endpoint",261 "GET /metadata returns name and description",262 metadata_ok,263 expected={"status_code": 200, "fields": ["name", "description"]},264 actual={265 "status_code": metadata_response.status_code,266 "name": (267 metadata_json.get("name")268 if isinstance(metadata_json, dict)269 else None270 ),271 "description": (272 metadata_json.get("description")273 if isinstance(metadata_json, dict)274 else None275 ),276 },277 )278 )279 280 # Criterion: Schema endpoint returns action/observation/state.281 try:282 schema_response = requests.get(f"{normalized_url}/schema", timeout=timeout_s)283 except requests.RequestException as exc:284 criteria.append(285 _make_criterion(286 "schema_endpoint",287 "GET /schema returns action, observation, and state schemas",288 False,289 details=f"Request failed: {type(exc).__name__}: {exc}",290 expected={291 "status_code": 200,292 "fields": ["action", "observation", "state"],293 },294 )295 )296 else:297 try:298 schema_json = schema_response.json()299 except ValueError:300 schema_json = None301 302 schema_ok = (303 schema_response.status_code == 200304 and isinstance(schema_json, dict)305 and isinstance(schema_json.get("action"), dict)306 and isinstance(schema_json.get("observation"), dict)307 and isinstance(schema_json.get("state"), dict)308 )309 criteria.append(310 _make_criterion(311 "schema_endpoint",312 "GET /schema returns action, observation, and state schemas",313 schema_ok,314 expected={315 "status_code": 200,316 "fields": ["action", "observation", "state"],317 },318 actual={319 "status_code": schema_response.status_code,320 "has_action": (321 isinstance(schema_json.get("action"), dict)322 if isinstance(schema_json, dict)323 else False324 ),325 "has_observation": (326 isinstance(schema_json.get("observation"), dict)327 if isinstance(schema_json, dict)328 else False329 ),330 "has_state": (331 isinstance(schema_json.get("state"), dict)332 if isinstance(schema_json, dict)333 else False334 ),335 },336 )337 )338 339 # Criterion: MCP endpoint is reachable.340 try:341 mcp_response = requests.post(342 f"{normalized_url}/mcp", json={}, timeout=timeout_s343 )344 except requests.RequestException as exc:345 criteria.append(346 _make_criterion(347 "mcp_endpoint",348 "POST /mcp is reachable and returns JSON-RPC payload",349 False,350 details=f"Request failed: {type(exc).__name__}: {exc}",351 expected={"status_code": 200, "jsonrpc": "2.0"},352 )353 )354 else:355 try:356 mcp_json = mcp_response.json()357 except ValueError:358 mcp_json = None359 360 mcp_ok = (361 mcp_response.status_code == 200362 and isinstance(mcp_json, dict)363 and mcp_json.get("jsonrpc") == "2.0"364 )365 criteria.append(366 _make_criterion(367 "mcp_endpoint",368 "POST /mcp is reachable and returns JSON-RPC payload",369 mcp_ok,370 expected={"status_code": 200, "jsonrpc": "2.0"},371 actual={372 "status_code": mcp_response.status_code,373 "jsonrpc": (374 mcp_json.get("jsonrpc") if isinstance(mcp_json, dict) else None375 ),376 },377 )378 )379 380 # Criterion: mode endpoint contract consistency via OpenAPI paths.381 if isinstance(openapi_paths, dict) and openapi_paths:382 has_reset = "/reset" in openapi_paths383 has_step = "/step" in openapi_paths384 has_state = "/state" in openapi_paths385 386 if has_reset:387 report["mode"] = "simulation"388 mode_ok = has_step and has_state389 expected_paths = {"/reset": True, "/step": True, "/state": True}390 else:391 report["mode"] = "production"392 mode_ok = not has_step and not has_state393 expected_paths = {"/reset": False, "/step": False, "/state": False}394 395 criteria.append(396 _make_criterion(397 "mode_endpoint_consistency",398 "OpenAPI endpoint set matches OpenEnv mode contract",399 mode_ok,400 expected=expected_paths,401 actual={402 "/reset": has_reset,403 "/step": has_step,404 "/state": has_state,405 },406 )407 )408 else:409 criteria.append(410 _make_criterion(411 "mode_endpoint_consistency",412 "OpenAPI endpoint set matches OpenEnv mode contract",413 False,414 details="Cannot determine mode without OpenAPI paths",415 expected={"openapi.paths": "present"},416 actual={"openapi.paths": "missing"},417 )418 )419 420 report["passed"] = all(421 criterion["passed"] for criterion in criteria if criterion.get("required", True)422 )423 report["summary"] = _build_summary(criteria)424 return report425 426 427def _has_main_guard_call(app_content: str) -> bool:428 """Return True when the module calls main() under a __main__ guard."""429 try:430 tree = ast.parse(app_content)431 except SyntaxError:432 return (433 "__name__" in app_content434 and "__main__" in app_content435 and "main(" in app_content436 )437 438 for node in ast.iter_child_nodes(tree):439 if not isinstance(node, ast.If) or not _is_main_guard(node.test):440 continue441 442 if any(_contains_main_call(guarded_node) for guarded_node in node.body):443 return True444 445 return False446 447 448def _is_main_guard(test: ast.expr) -> bool:449 """Return True for `if __name__ == "__main__"` tests."""450 return (451 isinstance(test, ast.Compare)452 and isinstance(test.left, ast.Name)453 and test.left.id == "__name__"454 and len(test.ops) == 1455 and isinstance(test.ops[0], ast.Eq)456 and len(test.comparators) == 1457 and isinstance(test.comparators[0], ast.Constant)458 and test.comparators[0].value == "__main__"459 )460 461 462def _contains_main_call(node: ast.AST) -> bool:463 """Return True when an AST node contains a direct `main(...)` call."""464 return any(465 isinstance(candidate, ast.Call)466 and isinstance(candidate.func, ast.Name)467 and candidate.func.id == "main"468 for candidate in ast.walk(node)469 )470 471 472_OPENENV_RUNTIME_DEP_RE = re.compile(r"^openenv(?:\s*(?:$|[<>=!~@;])|\[)")473_LEGACY_OPENENV_CORE_DEP_RE = re.compile(r"^openenv-core(?:\s*(?:$|[<>=!~@;])|\[)")474_OPENENV_DOCKER_INSTALL_RE = re.compile(475 r"(?<![a-z0-9_.-])openenv(?:\s*(?:$|[<>=!~@;])|\[)"476)477 478 479def _dockerfile_installs_openenv_runtime(env_path: Path) -> bool:480 """Return True when a Docker deployment installs OpenEnv outside pyproject."""481 for dockerfile_path in (482 env_path / "server" / "Dockerfile",483 env_path / "Dockerfile",484 ):485 if not dockerfile_path.exists():486 continue487 488 try:489 dockerfile = dockerfile_path.read_text(encoding="utf-8")490 except OSError:491 continue492 493 for line in dockerfile.splitlines():494 stripped = line.strip().lower()495 if not stripped or stripped.startswith("#"):496 continue497 if _OPENENV_DOCKER_INSTALL_RE.search(stripped):498 return True499 if "openenv-core" in stripped:500 return True501 502 return False503 504 505def validate_multi_mode_deployment(env_path: Path) -> tuple[bool, list[str]]:506 """507 Validate that an environment is ready for multi-mode deployment.508 509 Checks:510 1. pyproject.toml exists511 2. uv.lock exists512 3. pyproject.toml has [project.scripts] with server entry point513 4. server/app.py has a main() function514 5. Required dependencies are present515 516 Returns:517 `tuple` of `(is_valid, issues)` where `is_valid` is a `bool` and `issues` is a518 `list` of issue strings found during validation.519 """520 issues = []521 522 # Check pyproject.toml exists523 pyproject_path = env_path / "pyproject.toml"524 if not pyproject_path.exists():525 issues.append("Missing pyproject.toml")526 return False, issues527 528 # Check uv.lock exists529 lockfile_path = env_path / "uv.lock"530 if not lockfile_path.exists():531 issues.append("Missing uv.lock - run 'uv lock' to generate it")532 533 # Parse pyproject.toml534 try:535 with open(pyproject_path, "rb") as f:536 pyproject = tomllib.load(f)537 except Exception as e:538 issues.append(f"Failed to parse pyproject.toml: {e}")539 return False, issues540 541 # Check [project.scripts] section542 scripts = pyproject.get("project", {}).get("scripts", {})543 if "server" not in scripts:544 issues.append("Missing [project.scripts] server entry point")545 546 # Check server entry point format547 server_entry = scripts.get("server", "")548 if server_entry and ":main" not in server_entry:549 issues.append(550 f"Server entry point should reference main function, got: {server_entry}"551 )552 553 # Check required dependencies554 deps = [dep.lower() for dep in pyproject.get("project", {}).get("dependencies", [])]555 has_openenv = any(_OPENENV_RUNTIME_DEP_RE.match(dep) for dep in deps)556 has_legacy_core = any(_LEGACY_OPENENV_CORE_DEP_RE.match(dep) for dep in deps)557 has_dockerfile_core = _dockerfile_installs_openenv_runtime(env_path)558 559 if not (has_openenv or has_legacy_core or has_dockerfile_core):560 issues.append("Missing required dependency: openenv>=0.2.0")561 562 # Check server/app.py exists563 server_app = env_path / "server" / "app.py"564 if not server_app.exists():565 issues.append("Missing server/app.py")566 else:567 # Check for main() function (flexible - with or without parameters)568 app_content = server_app.read_text(encoding="utf-8")569 if "def main(" not in app_content:570 issues.append("server/app.py missing main() function")571 572 # Check if main() is callable573 if not _has_main_guard_call(app_content):574 issues.append(575 "server/app.py main() function not callable (missing if __name__ == '__main__')"576 )577 578 return len(issues) == 0, issues579 580 581def get_deployment_modes(env_path: Path) -> dict[str, bool]:582 """583 Check which deployment modes are supported by the environment.584 585 Returns:586 `dict` mapping deployment mode names to whether they are supported.587 """588 modes = {589 "docker": False,590 "openenv_serve": False,591 "uv_run": False,592 "python_module": False,593 }594 595 # Check Docker (Dockerfile may be in server/ or at env root)596 modes["docker"] = (env_path / "server" / "Dockerfile").exists() or (597 env_path / "Dockerfile"598 ).exists()599 600 # Check multi-mode deployment readiness601 is_valid, _ = validate_multi_mode_deployment(env_path)602 if is_valid:603 modes["openenv_serve"] = True604 modes["uv_run"] = True605 modes["python_module"] = True606 607 return modes608 609 610def format_validation_report(env_name: str, is_valid: bool, issues: list[str]) -> str:611 """612 Format a validation report for display.613 614 Returns:615 `str`: formatted validation report.616 """617 if is_valid:618 return f"[OK] {env_name}: Ready for multi-mode deployment"619 620 report = [f"[FAIL] {env_name}: Not ready for multi-mode deployment", ""]621 report.append("Issues found:")622 for issue in issues:623 report.append(f" - {issue}")624 625 return "\n".join(report)626 627 628def build_local_validation_json_report(629 env_name: str,630 env_path: Path,631 is_valid: bool,632 issues: list[str],633 deployment_modes: dict[str, bool] | None = None,634) -> dict[str, Any]:635 """Build a JSON report for local environment validation."""636 criteria = [637 _make_criterion(638 "multi_mode_deployment_readiness",639 "Environment structure is ready for multi-mode deployment",640 is_valid,641 details="No issues found" if is_valid else f"{len(issues)} issue(s) found",642 actual={"issues": issues},643 )644 ]645 646 if deployment_modes:647 for mode, supported in deployment_modes.items():648 criteria.append(649 _make_criterion(650 f"deployment_mode_{mode}",651 f"Deployment mode '{mode}' is supported",652 supported,653 required=False,654 )655 )656 657 return {658 "target": str(env_path),659 "environment": env_name,660 "validation_type": "local_environment",661 "standard_version": "local",662 "standard_profile": "openenv-local",663 "passed": is_valid,664 "summary": _build_summary(criteria),665 "criteria": criteria,666 "issues": issues,667 "deployment_modes": deployment_modes or {},668 }669