openenv/coding_env
21
1# Copyright (c) Meta Platforms, Inc. and affiliates.2# All rights reserved.3#4# This source code is licensed under the BSD-style license found in the5# LICENSE file in the root directory of this source tree.6 7"""8Validation utilities for multi-mode deployment readiness.9 10This module provides functions to check if environments are properly11configured for multi-mode deployment (Docker, direct Python, notebooks, clusters).12"""13 14from pathlib import Path15from typing import Any16from urllib.parse import urlparse17 18import requests19 20try:21 import tomllib22except ModuleNotFoundError:23 import tomli as tomllib24 25 26def _make_criterion(27 criterion_id: str,28 description: str,29 passed: bool,30 *,31 required: bool = True,32 details: str | None = None,33 expected: Any | None = None,34 actual: Any | None = None,35) -> dict[str, Any]:36 """Create a standard criterion result payload."""37 criterion: dict[str, Any] = {38 "id": criterion_id,39 "description": description,40 "passed": passed,41 "required": required,42 }43 if details is not None:44 criterion["details"] = details45 if expected is not None:46 criterion["expected"] = expected47 if actual is not None:48 criterion["actual"] = actual49 return criterion50 51 52def _normalize_runtime_url(base_url: str) -> str:53 """Normalize and validate a runtime target URL."""54 target = base_url.strip()55 if not target:56 raise ValueError("Runtime URL cannot be empty")57 58 if "://" not in target:59 target = f"http://{target}"60 61 parsed = urlparse(target)62 if not parsed.scheme or not parsed.netloc:63 raise ValueError(f"Invalid runtime URL: {base_url}")64 65 return target.rstrip("/")66 67 68def _runtime_standard_profile(api_version: str) -> str:69 """Resolve the runtime standard profile for an API version."""70 if api_version.startswith("1."):71 return "openenv-http/1.x"72 return "openenv-http/unknown"73 74 75def _build_summary(criteria: list[dict[str, Any]]) -> dict[str, Any]:76 """Build a compact pass/fail summary for a criteria list."""77 total_count = len(criteria)78 passed_count = sum(1 for criterion in criteria if criterion.get("passed", False))79 failed_criteria = [80 criterion.get("id", "unknown")81 for criterion in criteria82 if not criterion.get("passed", False)83 ]84 required_criteria = [85 criterion for criterion in criteria if criterion.get("required", True)86 ]87 required_total_count = len(required_criteria)88 required_passed_count = sum(89 1 for criterion in required_criteria if criterion.get("passed", False)90 )91 92 return {93 "passed_count": passed_count,94 "total_count": total_count,95 "failed_criteria": failed_criteria,96 "required_passed_count": required_passed_count,97 "required_total_count": required_total_count,98 }99 100 101def validate_running_environment(102 base_url: str, timeout_s: float = 5.0103) -> dict[str, Any]:104 """105 Validate a running OpenEnv server against runtime API standards.106 107 The returned JSON report contains an overall pass/fail result and108 per-criterion outcomes that can be consumed in CI.109 """110 normalized_url = _normalize_runtime_url(base_url)111 criteria: list[dict[str, Any]] = []112 113 report: dict[str, Any] = {114 "target": normalized_url,115 "validation_type": "running_environment",116 "standard_version": "unknown",117 "standard_profile": "openenv-http/unknown",118 "mode": "unknown",119 "passed": False,120 "summary": {},121 "criteria": criteria,122 }123 124 openapi_paths: dict[str, Any] = {}125 api_version = "unknown"126 127 # Criterion: OpenAPI endpoint reachable with a declared version.128 try:129 openapi_response = requests.get(130 f"{normalized_url}/openapi.json", timeout=timeout_s131 )132 except requests.RequestException as exc:133 criteria.append(134 _make_criterion(135 "openapi_version_available",136 "GET /openapi.json returns OpenAPI info.version",137 False,138 details=f"Request failed: {type(exc).__name__}: {exc}",139 expected={"status_code": 200, "info.version": "string"},140 )141 )142 else:143 try:144 openapi_json = openapi_response.json()145 except ValueError:146 openapi_json = None147 148 openapi_ok = (149 openapi_response.status_code == 200150 and isinstance(openapi_json, dict)151 and isinstance(openapi_json.get("info"), dict)152 and isinstance(openapi_json["info"].get("version"), str)153 )154 155 if openapi_ok:156 api_version = str(openapi_json["info"]["version"])157 openapi_paths = openapi_json.get("paths", {})158 criteria.append(159 _make_criterion(160 "openapi_version_available",161 "GET /openapi.json returns OpenAPI info.version",162 True,163 expected={"status_code": 200, "info.version": "string"},164 actual={165 "status_code": openapi_response.status_code,166 "info.version": api_version,167 },168 )169 )170 else:171 criteria.append(172 _make_criterion(173 "openapi_version_available",174 "GET /openapi.json returns OpenAPI info.version",175 False,176 details="Response missing required OpenAPI info.version field",177 expected={"status_code": 200, "info.version": "string"},178 actual={179 "status_code": openapi_response.status_code,180 "body_type": (181 type(openapi_json).__name__182 if openapi_json is not None183 else "non_json"184 ),185 },186 )187 )188 189 report["standard_version"] = api_version190 report["standard_profile"] = _runtime_standard_profile(api_version)191 192 # Criterion: Health endpoint.193 try:194 health_response = requests.get(f"{normalized_url}/health", timeout=timeout_s)195 except requests.RequestException as exc:196 criteria.append(197 _make_criterion(198 "health_endpoint",199 "GET /health returns healthy status",200 False,201 details=f"Request failed: {type(exc).__name__}: {exc}",202 expected={"status_code": 200, "status": "healthy"},203 )204 )205 else:206 try:207 health_json = health_response.json()208 except ValueError:209 health_json = None210 211 health_ok = (212 health_response.status_code == 200213 and isinstance(health_json, dict)214 and health_json.get("status") == "healthy"215 )216 criteria.append(217 _make_criterion(218 "health_endpoint",219 "GET /health returns healthy status",220 health_ok,221 expected={"status_code": 200, "status": "healthy"},222 actual={223 "status_code": health_response.status_code,224 "status": (225 health_json.get("status")226 if isinstance(health_json, dict)227 else None228 ),229 },230 )231 )232 233 # Criterion: Metadata endpoint has required fields.234 try:235 metadata_response = requests.get(236 f"{normalized_url}/metadata", timeout=timeout_s237 )238 except requests.RequestException as exc:239 criteria.append(240 _make_criterion(241 "metadata_endpoint",242 "GET /metadata returns name and description",243 False,244 details=f"Request failed: {type(exc).__name__}: {exc}",245 expected={"status_code": 200, "fields": ["name", "description"]},246 )247 )248 else:249 try:250 metadata_json = metadata_response.json()251 except ValueError:252 metadata_json = None253 254 metadata_ok = (255 metadata_response.status_code == 200256 and isinstance(metadata_json, dict)257 and isinstance(metadata_json.get("name"), str)258 and isinstance(metadata_json.get("description"), str)259 )260 criteria.append(261 _make_criterion(262 "metadata_endpoint",263 "GET /metadata returns name and description",264 metadata_ok,265 expected={"status_code": 200, "fields": ["name", "description"]},266 actual={267 "status_code": metadata_response.status_code,268 "name": (269 metadata_json.get("name")270 if isinstance(metadata_json, dict)271 else None272 ),273 "description": (274 metadata_json.get("description")275 if isinstance(metadata_json, dict)276 else None277 ),278 },279 )280 )281 282 # Criterion: Schema endpoint returns action/observation/state.283 try:284 schema_response = requests.get(f"{normalized_url}/schema", timeout=timeout_s)285 except requests.RequestException as exc:286 criteria.append(287 _make_criterion(288 "schema_endpoint",289 "GET /schema returns action, observation, and state schemas",290 False,291 details=f"Request failed: {type(exc).__name__}: {exc}",292 expected={293 "status_code": 200,294 "fields": ["action", "observation", "state"],295 },296 )297 )298 else:299 try:300 schema_json = schema_response.json()301 except ValueError:302 schema_json = None303 304 schema_ok = (305 schema_response.status_code == 200306 and isinstance(schema_json, dict)307 and isinstance(schema_json.get("action"), dict)308 and isinstance(schema_json.get("observation"), dict)309 and isinstance(schema_json.get("state"), dict)310 )311 criteria.append(312 _make_criterion(313 "schema_endpoint",314 "GET /schema returns action, observation, and state schemas",315 schema_ok,316 expected={317 "status_code": 200,318 "fields": ["action", "observation", "state"],319 },320 actual={321 "status_code": schema_response.status_code,322 "has_action": (323 isinstance(schema_json.get("action"), dict)324 if isinstance(schema_json, dict)325 else False326 ),327 "has_observation": (328 isinstance(schema_json.get("observation"), dict)329 if isinstance(schema_json, dict)330 else False331 ),332 "has_state": (333 isinstance(schema_json.get("state"), dict)334 if isinstance(schema_json, dict)335 else False336 ),337 },338 )339 )340 341 # Criterion: MCP endpoint is reachable.342 try:343 mcp_response = requests.post(344 f"{normalized_url}/mcp", json={}, timeout=timeout_s345 )346 except requests.RequestException as exc:347 criteria.append(348 _make_criterion(349 "mcp_endpoint",350 "POST /mcp is reachable and returns JSON-RPC payload",351 False,352 details=f"Request failed: {type(exc).__name__}: {exc}",353 expected={"status_code": 200, "jsonrpc": "2.0"},354 )355 )356 else:357 try:358 mcp_json = mcp_response.json()359 except ValueError:360 mcp_json = None361 362 mcp_ok = (363 mcp_response.status_code == 200364 and isinstance(mcp_json, dict)365 and mcp_json.get("jsonrpc") == "2.0"366 )367 criteria.append(368 _make_criterion(369 "mcp_endpoint",370 "POST /mcp is reachable and returns JSON-RPC payload",371 mcp_ok,372 expected={"status_code": 200, "jsonrpc": "2.0"},373 actual={374 "status_code": mcp_response.status_code,375 "jsonrpc": (376 mcp_json.get("jsonrpc") if isinstance(mcp_json, dict) else None377 ),378 },379 )380 )381 382 # Criterion: mode endpoint contract consistency via OpenAPI paths.383 if isinstance(openapi_paths, dict) and openapi_paths:384 has_reset = "/reset" in openapi_paths385 has_step = "/step" in openapi_paths386 has_state = "/state" in openapi_paths387 388 if has_reset:389 report["mode"] = "simulation"390 mode_ok = has_step and has_state391 expected_paths = {"/reset": True, "/step": True, "/state": True}392 else:393 report["mode"] = "production"394 mode_ok = not has_step and not has_state395 expected_paths = {"/reset": False, "/step": False, "/state": False}396 397 criteria.append(398 _make_criterion(399 "mode_endpoint_consistency",400 "OpenAPI endpoint set matches OpenEnv mode contract",401 mode_ok,402 expected=expected_paths,403 actual={404 "/reset": has_reset,405 "/step": has_step,406 "/state": has_state,407 },408 )409 )410 else:411 criteria.append(412 _make_criterion(413 "mode_endpoint_consistency",414 "OpenAPI endpoint set matches OpenEnv mode contract",415 False,416 details="Cannot determine mode without OpenAPI paths",417 expected={"openapi.paths": "present"},418 actual={"openapi.paths": "missing"},419 )420 )421 422 report["passed"] = all(423 criterion["passed"] for criterion in criteria if criterion.get("required", True)424 )425 report["summary"] = _build_summary(criteria)426 return report427 428 429def validate_multi_mode_deployment(env_path: Path) -> tuple[bool, list[str]]:430 """431 Validate that an environment is ready for multi-mode deployment.432 433 Checks:434 1. pyproject.toml exists435 2. uv.lock exists436 3. pyproject.toml has [project.scripts] with server entry point437 4. server/app.py has a main() function438 5. Required dependencies are present439 440 Returns:441 Tuple of (is_valid, list of issues found)442 """443 issues = []444 445 # Check pyproject.toml exists446 pyproject_path = env_path / "pyproject.toml"447 if not pyproject_path.exists():448 issues.append("Missing pyproject.toml")449 return False, issues450 451 # Check uv.lock exists452 lockfile_path = env_path / "uv.lock"453 if not lockfile_path.exists():454 issues.append("Missing uv.lock - run 'uv lock' to generate it")455 456 # Parse pyproject.toml457 try:458 with open(pyproject_path, "rb") as f:459 pyproject = tomllib.load(f)460 except Exception as e:461 issues.append(f"Failed to parse pyproject.toml: {e}")462 return False, issues463 464 # Check [project.scripts] section465 scripts = pyproject.get("project", {}).get("scripts", {})466 if "server" not in scripts:467 issues.append("Missing [project.scripts] server entry point")468 469 # Check server entry point format470 server_entry = scripts.get("server", "")471 if server_entry and ":main" not in server_entry:472 issues.append(473 f"Server entry point should reference main function, got: {server_entry}"474 )475 476 # Check required dependencies477 deps = [dep.lower() for dep in pyproject.get("project", {}).get("dependencies", [])]478 has_openenv = any(479 dep.startswith("openenv") and not dep.startswith("openenv-core") for dep in deps480 )481 has_legacy_core = any(dep.startswith("openenv-core") for dep in deps)482 483 if not (has_openenv or has_legacy_core):484 issues.append(485 "Missing required dependency: openenv-core>=0.2.0 (or openenv>=0.2.0)"486 )487 488 # Check server/app.py exists489 server_app = env_path / "server" / "app.py"490 if not server_app.exists():491 issues.append("Missing server/app.py")492 else:493 # Check for main() function (flexible - with or without parameters)494 app_content = server_app.read_text(encoding="utf-8")495 if "def main(" not in app_content:496 issues.append("server/app.py missing main() function")497 498 # Check if main() is callable499 if "__name__" not in app_content or "main()" not in app_content:500 issues.append(501 "server/app.py main() function not callable (missing if __name__ == '__main__')"502 )503 504 return len(issues) == 0, issues505 506 507def get_deployment_modes(env_path: Path) -> dict[str, bool]:508 """509 Check which deployment modes are supported by the environment.510 511 Returns:512 Dictionary with deployment mode names and whether they're supported513 """514 modes = {515 "docker": False,516 "openenv_serve": False,517 "uv_run": False,518 "python_module": False,519 }520 521 # Check Docker (Dockerfile may be in server/ or at env root)522 modes["docker"] = (env_path / "server" / "Dockerfile").exists() or (523 env_path / "Dockerfile"524 ).exists()525 526 # Check multi-mode deployment readiness527 is_valid, _ = validate_multi_mode_deployment(env_path)528 if is_valid:529 modes["openenv_serve"] = True530 modes["uv_run"] = True531 modes["python_module"] = True532 533 return modes534 535 536def format_validation_report(env_name: str, is_valid: bool, issues: list[str]) -> str:537 """538 Format a validation report for display.539 540 Returns:541 Formatted report string542 """543 if is_valid:544 return f"[OK] {env_name}: Ready for multi-mode deployment"545 546 report = [f"[FAIL] {env_name}: Not ready for multi-mode deployment", ""]547 report.append("Issues found:")548 for issue in issues:549 report.append(f" - {issue}")550 551 return "\n".join(report)552 553 554def build_local_validation_json_report(555 env_name: str,556 env_path: Path,557 is_valid: bool,558 issues: list[str],559 deployment_modes: dict[str, bool] | None = None,560) -> dict[str, Any]:561 """Build a JSON report for local environment validation."""562 criteria = [563 _make_criterion(564 "multi_mode_deployment_readiness",565 "Environment structure is ready for multi-mode deployment",566 is_valid,567 details="No issues found" if is_valid else f"{len(issues)} issue(s) found",568 actual={"issues": issues},569 )570 ]571 572 if deployment_modes:573 for mode, supported in deployment_modes.items():574 criteria.append(575 _make_criterion(576 f"deployment_mode_{mode}",577 f"Deployment mode '{mode}' is supported",578 supported,579 required=False,580 )581 )582 583 return {584 "target": str(env_path),585 "environment": env_name,586 "validation_type": "local_environment",587 "standard_version": "local",588 "standard_profile": "openenv-local",589 "passed": is_valid,590 "summary": _build_summary(criteria),591 "criteria": criteria,592 "issues": issues,593 "deployment_modes": deployment_modes or {},594 }595