openenv/echo_env
6
1"""Validation orchestration: parse → grade → apply policy → report."""2 3import hashlib4import os5import stat6import time7import uuid8from dataclasses import replace9from pathlib import Path10 11from .graders import GraderRegistry, Subject12from .graders.runtime import (13 ObservationSchemaGrader,14 RewardWellFormedGrader,15 StateContractGrader,16)17from .graders.static import StaticManifestGrader18from .manifest import ManifestError, NormalizedManifest, NormalizedManifestV219from .parsers import ParserRegistry20from .parsers.openenv_yaml import OpenEnvYamlParser21from .policy import apply_policy, load_policy, PolicyError, SeverityPolicy22from .providers import ProviderError, StartupError, UnsupportedCapability23from .report import CheckResult, ValidationReport, ValidationReportV224from .runtime.artifacts import write_runtime_bundle25from .runtime.collector import collect_runtime_evidence, RuntimeCollectionInterrupted26from .runtime.contracts import LaunchSpec, load_runtime_plan, RuntimePlanError27from .runtime.scheduler import execute_graders, order_graders28from .signature import detect_signature29from .types import CheckStatus, Lane, Level, ProviderCapability30 31REPORT_SCHEMA_VERSION = "1"32 33_DIGEST_EXCLUDED_DIRS = {".git", "__pycache__", ".venv", ".worktrees", "outputs"}34 35 36def source_digest(package_root: Path) -> str:37 """38 Deterministic sha256 over the package tree (relative paths + file contents).39 40 Args:41 package_root (`Path`):42 The package directory.43 44 Returns:45 `str`: a 64-character hex digest.46 """47 digest = hashlib.sha256()48 files = []49 for path in package_root.rglob("*"):50 relative_path = path.relative_to(package_root)51 if any(part in _DIGEST_EXCLUDED_DIRS for part in relative_path.parts):52 continue53 info = path.lstat()54 if stat.S_ISLNK(info.st_mode):55 raise ValueError("validation source may not contain symbolic links")56 if stat.S_ISREG(info.st_mode):57 files.append((relative_path.as_posix(), path, info))58 elif not stat.S_ISDIR(info.st_mode):59 raise ValueError("validation source must contain regular files")60 61 for relative_path, path, info in sorted(files, key=lambda item: item[0]):62 digest.update(relative_path.encode())63 digest.update(b"\0")64 flags = (65 os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)66 )67 fd = os.open(path, flags)68 with os.fdopen(fd, "rb") as source:69 opened = os.fstat(source.fileno())70 if not stat.S_ISREG(opened.st_mode):71 raise ValueError("validation source must contain regular files")72 if not os.path.samestat(info, opened):73 raise ValueError("validation source changed before it could be read")74 while chunk := source.read(1024 * 1024):75 digest.update(chunk)76 digest.update(b"\0")77 return digest.hexdigest()78 79 80def default_parser_registry() -> ParserRegistry:81 """The parsers shipped in this build."""82 registry = ParserRegistry()83 registry.register(OpenEnvYamlParser())84 return registry85 86 87def default_grader_registry(policy: SeverityPolicy) -> GraderRegistry:88 """The graders shipped in this build, including their selection metadata."""89 registry = GraderRegistry()90 registry.register(StaticManifestGrader(policy.bounds))91 registry.register(RewardWellFormedGrader())92 registry.register(ObservationSchemaGrader())93 registry.register(StateContractGrader())94 return registry95 96 97def _outcome(check_id, status, reason, *, started=None, measured=None):98 return CheckResult(99 check_id=check_id,100 status=status,101 evidence=[reason],102 measured=measured or {},103 duration_s=time.monotonic() - started if started is not None else 0,104 )105 106 107def _applicable(check_id, manifest):108 if manifest is None:109 return True110 if check_id in {"runtime.rubric_introspectable", "runtime.reward_attribution"}:111 return manifest.capabilities.rubric_tree112 if check_id == "runtime.task_declaration_accuracy":113 return manifest.capabilities.task_api or bool(114 manifest.capabilities.declared_task_count115 )116 return True117 118 119def _runtime(subject, graders, *, skip_build, provider):120 """Own build/start/collection/stop and retain failure evidence through teardown."""121 manifest = subject.manifest122 plan = None123 evidence = None124 running = None125 inspection = {}126 cleanup = {"required": False, "completed": True}127 started = time.monotonic()128 attempted = False129 result = None130 checks = []131 try:132 if skip_build:133 raise UnsupportedCapability(134 "--skip-build: build-dependent runtime checks skipped"135 )136 if not isinstance(manifest, NormalizedManifestV2):137 raise UnsupportedCapability(138 "missing validation.execution declaration and runtime plan"139 )140 plan = load_runtime_plan(subject.root, manifest.execution)141 if manifest.execution.requires_credentials:142 raise UnsupportedCapability(143 "credential_delivery is deferred for this release"144 )145 if provider is None:146 from .providers.docker import DockerValidationProvider147 148 provider = DockerValidationProvider()149 if manifest.network.mode not in provider.supported_network_modes:150 raise UnsupportedCapability(151 f"provider cannot enforce network mode {manifest.network.mode}"152 )153 if (154 manifest.resources.gpus155 and ProviderCapability.GPU not in provider.capabilities156 ):157 raise UnsupportedCapability("missing provider capability: gpu")158 if ProviderCapability.IMAGE_BUILD not in provider.capabilities:159 raise UnsupportedCapability("missing provider capability: image_build")160 # Validate all launch constraints before any container or build is started.161 spec = LaunchSpec(162 image_ref="sha256:" + "0" * 64,163 resources=manifest.resources,164 network=manifest.network,165 run_id="validation-" + uuid.uuid4().hex,166 )167 attempted = True168 image_ref = provider.build(subject.root, manifest.execution)169 running = provider.start(170 LaunchSpec.model_validate({**spec.model_dump(), "image_ref": image_ref})171 )172 cleanup = {"required": True, "completed": False}173 inspection = running.inspect()174 result = _outcome(175 "runtime.startup",176 CheckStatus.PASS,177 "subject built and reached its control endpoint",178 started=started,179 measured={"provider": provider.name, "image_ref": image_ref},180 )181 evidence = collect_runtime_evidence(182 running.base_url,183 plan,184 episode_timeout_s=manifest.resources.episode_timeout_s,185 )186 # Protocol collection must finish before its dependent contract checks run.187 if evidence.failure_reason:188 result = _outcome(189 "runtime.startup",190 CheckStatus.FAIL,191 evidence.failure_reason,192 started=started,193 )194 subject = replace(195 subject, image_ref=image_ref, running=running, runtime_evidence=evidence196 )197 checks = execute_graders(198 graders,199 subject,200 provider_capabilities=provider.capabilities,201 prior=[result],202 )203 except UnsupportedCapability as exc:204 result = _outcome(205 "runtime.startup", CheckStatus.SKIP, str(exc), started=started206 )207 except (RuntimePlanError, StartupError) as exc:208 result = _outcome(209 "runtime.startup",210 CheckStatus.FAIL,211 str(exc)[:4096],212 started=started,213 )214 except ProviderError as exc:215 result = _outcome(216 "runtime.startup",217 CheckStatus.ERROR,218 str(exc)[:4096],219 started=started,220 )221 except KeyboardInterrupt as exc:222 if isinstance(exc, RuntimeCollectionInterrupted):223 evidence = exc.evidence224 result = _outcome(225 "runtime.startup",226 CheckStatus.ERROR,227 "validation interrupted",228 started=started,229 )230 except Exception as exc:231 result = _outcome(232 "runtime.startup",233 CheckStatus.ERROR,234 f"runtime orchestration failed ({type(exc).__name__})",235 started=started,236 )237 finally:238 if running is not None:239 try:240 running.stop()241 cleanup["completed"] = True242 except (Exception, KeyboardInterrupt) as exc:243 cleanup["completed"] = False244 cleanup["reason"] = f"subject teardown failed ({type(exc).__name__})"245 if result is None:246 result = _outcome(247 "runtime.startup",248 CheckStatus.ERROR,249 "subject teardown failed",250 started=started,251 )252 else:253 result.status = CheckStatus.ERROR254 result.evidence.append("subject teardown failed")255 result.duration_s = time.monotonic() - started256 return [result, *checks], attempted, plan, evidence, inspection, cleanup257 258 259def run_validation(260 target: Path,261 *,262 max_level: Level = Level.SEMANTIC,263 skip_build: bool = False,264 policy: SeverityPolicy | None = None,265 provider=None,266 artifacts_dir: Path | None = None,267) -> ValidationReport | ValidationReportV2:268 """269 Validate a package end to end and return the report.270 271 Raises [`~openenv.validation.signature.SignatureError`] for ambiguous or272 unrecognized packages and273 [`~openenv.validation.signature.UnsupportedPackageError`] for274 recognized-but-unsupported ones (CLI exit code 2). A package whose declarations275 fail the manifest schema yields a normal report with a `static.manifest` FAIL276 (exit code 1).277 278 Args:279 target (`Path`):280 The package directory.281 max_level ([`~openenv.validation.types.Level`], *optional*, defaults to `Level.SEMANTIC`):282 Level ceiling; graders above it are not selected.283 skip_build (`bool`, *optional*, defaults to `False`):284 Skip the image build; build-dependent checks SKIP with a reason.285 policy ([`~openenv.validation.policy.SeverityPolicy`], *optional*):286 `None` chooses v1 for static and v2 for runtime/semantic ceilings.287 provider ([`~openenv.validation.providers.ValidationProvider`], *optional*):288 Validation-only provider; defaults to Docker-local for runtime runs.289 artifacts_dir (`Path`, *optional*):290 Write a redacted reproduction bundle to this directory.291 292 Returns:293 [`~openenv.validation.report.ValidationReport`]: the completed report.294 """295 target = Path(target)296 wants_runtime = max_level >= Level.RUNTIME297 policy = policy or load_policy("v2" if wants_runtime else "v1")298 if wants_runtime and "runtime.startup" not in policy.entries_for_lane(Lane.LOCAL):299 raise PolicyError(300 "runtime validation requires policy v2; v1 supports --level static"301 )302 signature = detect_signature(target)303 try:304 digest_before = source_digest(target)305 except (ValueError, OSError):306 digest_before = ""307 308 parser = default_parser_registry().parser_for(signature)309 graders = default_grader_registry(policy)310 manifest: NormalizedManifest | None = None311 results: list[CheckResult] = []312 selected = {}313 314 parse_started = time.monotonic()315 if not digest_before:316 results.append(317 _outcome(318 "static.manifest",319 CheckStatus.ERROR,320 "package source could not be verified before validation",321 started=parse_started,322 )323 )324 else:325 try:326 manifest = parser.parse(target)327 except ManifestError as exc:328 results.append(329 CheckResult(330 check_id="static.manifest",331 status=CheckStatus.FAIL,332 measured={"schema_errors": len(exc.errors)},333 evidence=exc.errors,334 remediation=exc.remediation,335 duration_s=time.monotonic() - parse_started,336 )337 )338 339 levels = [Level.STATIC]340 plan = evidence = None341 inspection = {}342 cleanup = {"required": False, "completed": True}343 if manifest is not None:344 selected = {345 grader.check_id: grader for grader in graders.select(manifest, max_level)346 }347 subject = Subject(348 root=target,349 manifest=manifest,350 image_ref=None,351 running=None,352 outputs_dir=target / "outputs",353 )354 results.extend(355 execute_graders(356 [357 grader358 for grader in selected.values()359 if grader.level is Level.STATIC360 ],361 subject,362 )363 )364 if wants_runtime:365 runtime_results, attempted, plan, evidence, inspection, cleanup = _runtime(366 subject,367 [368 grader369 for grader in selected.values()370 if grader.level is Level.RUNTIME371 ],372 skip_build=skip_build,373 provider=provider,374 )375 results.extend(runtime_results)376 if attempted:377 levels.append(Level.RUNTIME)378 379 if wants_runtime:380 # Policy IDs are an inventory, not evidence that a grader exists. Keep the381 # incomplete surface explicit throughout the staged implementation.382 present = {result.check_id for result in results}383 for entry in policy.entries_for_lane(Lane.LOCAL).values():384 grader = graders.get(entry.check_id)385 applicable = (386 manifest is None387 or entry.check_id in selected388 or (grader is None and _applicable(entry.check_id, manifest))389 )390 if (391 entry.level in {Level.RUNTIME, Level.SEMANTIC}392 and entry.level <= max_level393 and entry.check_id not in present394 and applicable395 ):396 reason = "grader not implemented in this build"397 if manifest is None:398 reason = "unmet dependency: valid manifest"399 elif grader is not None and grader.depends_on:400 reason = "unmet dependency: " + ", ".join(grader.depends_on)401 results.append(_outcome(entry.check_id, CheckStatus.SKIP, reason))402 source_problem = None403 if not digest_before:404 source_problem = "package source could not be verified before validation"405 else:406 try:407 digest_after = source_digest(target)408 except (ValueError, OSError):409 digest_after = None410 if digest_after != digest_before:411 source_problem = (412 "package source changed during validation"413 if digest_after is not None414 else "package source could not be verified after validation"415 )416 if source_problem is not None:417 invalidated = {"runtime.startup"}418 for grader in order_graders(list(selected.values())):419 if invalidated.intersection(grader.depends_on):420 invalidated.add(grader.check_id)421 results = [422 _outcome(423 r.check_id,424 CheckStatus.SKIP,425 f"unmet dependency: runtime.startup ({source_problem})",426 )427 if r.check_id in invalidated428 else r429 for r in results430 if r.check_id != "runtime.startup"431 ]432 results.append(433 _outcome(434 "runtime.startup",435 CheckStatus.ERROR,436 source_problem,437 )438 )439 440 report_type = (441 ValidationReportV2442 if wants_runtime or isinstance(manifest, NormalizedManifestV2)443 else ValidationReport444 )445 report = report_type(446 report_schema_version="2"447 if report_type is ValidationReportV2448 else REPORT_SCHEMA_VERSION,449 target=str(target),450 source_digest=digest_before,451 signature=signature,452 manifest=manifest,453 policy_version=policy.policy_version,454 lane=Lane.LOCAL,455 levels_run=levels,456 results=results,457 verdict=apply_policy(results, policy, Lane.LOCAL),458 )459 if artifacts_dir is not None:460 write_runtime_bundle(461 artifacts_dir,462 report,463 plan=plan,464 evidence=evidence,465 provider=inspection,466 cleanup=cleanup,467 )468 return report469