openenv/coding_env

1# Copyright (c) Meta Platforms, Inc. and affiliates.2# All rights reserved.3#4# This source code is licensed under the BSD-style license found in the5# LICENSE file in the root directory of this source tree.6 7"""Inspect AI harness integration for OpenEnv.8 9Requires the ``inspect-ai`` package: ``pip install 'inspect-ai>=0.3.0'``10"""11 12from __future__ import annotations13 14from typing import Any, Dict, Optional15 16from openenv.core.evals.base import EvalHarness17 18 19class InspectAIHarness(EvalHarness):20    """Evaluation harness wrapping Inspect AI's ``eval()`` function.21 22    All ``inspect_ai`` imports are deferred to :meth:`run` so this class is23    importable without inspect-ai installed.  An ``ImportError`` with a clear24    message is raised at call time if the dependency is missing.25 26    Args:27        log_dir: Directory for evaluation log output. Defaults to None28            (Inspect AI writes logs to its default location).29 30    ``eval_parameters`` keys accepted by :meth:`run`:31 32    +--------------------------+----------+-----------------+-----------------------------------+33    | Key                      | Type     | Default         | Purpose                           |34    +==========================+==========+=================+===================================+35    | ``model``                | str      | *required*      | Model string, e.g. "openai/gpt-4o"|36    | ``task``                 | str|None | ``dataset`` arg | Task file path or task string     |37    | ``task_args``            | dict     | ``{}``          | Arguments to pass to the task     |38    | ``max_samples``          | int|None | None            | Limit samples per task            |39    | ``temperature``          | float|None| None           | Model generation temperature      |40    | ``max_tokens``           | int|None | None            | Max generation tokens             |41    | ``epochs``               | int|None | None            | Number of evaluation epochs       |42    | ``solver``               | list|None| None            | Solver pipeline override          |43    | ``scorer``               | list|None| None            | Scorer override                   |44    | ``model_args``           | dict     | ``{}``          | Provider-specific model kwargs    |45    +--------------------------+----------+-----------------+-----------------------------------+46    """47 48    def __init__(49        self,50        *,51        log_dir: Optional[str] = None,52    ):53        self.log_dir = log_dir54 55    def run(56        self,57        harness_version: str,58        library_versions: Dict[str, str],59        dataset: str,60        eval_parameters: Dict[str, Any],61    ) -> Dict[str, Any]:62        """Run an Inspect AI evaluation.63 64        Args:65            harness_version: Version of inspect-ai being used.66            library_versions: Versions of supporting libraries.67            dataset: Default task string (used when ``task`` is not specified68                in *eval_parameters*).69            eval_parameters: See class docstring for accepted keys.70 71        Returns:72            Dictionary mapping metric names to scores.73 74        Raises:75            ImportError: If ``inspect-ai`` is not installed.76            ValueError: If ``model`` is missing from *eval_parameters*.77            RuntimeError: If the evaluation fails (log status is not "success").78        """79        try:80            from inspect_ai import eval as inspect_eval81        except ImportError:82            raise ImportError(83                "inspect-ai is required for InspectAIHarness. "84                "Install it with: pip install 'inspect-ai>=0.3.0'"85            )86 87        # Extract required model parameter88        model = eval_parameters.get("model")89        if model is None:90            raise ValueError(91                "eval_parameters must include 'model' "92                "(e.g. 'openai/gpt-4o', 'hf/meta-llama/...')."93            )94 95        # Task: explicit parameter or fall back to dataset96        task = eval_parameters.get("task", dataset)97 98        # Build eval kwargs99        eval_kwargs: Dict[str, Any] = {}100 101        task_args = eval_parameters.get("task_args", {})102        if task_args:103            eval_kwargs["task_args"] = task_args104 105        model_args = eval_parameters.get("model_args", {})106        if model_args:107            eval_kwargs["model_args"] = model_args108 109        for key in ("max_samples", "temperature", "max_tokens", "epochs"):110            value = eval_parameters.get(key)111            if value is not None:112                eval_kwargs[key] = value113 114        if eval_parameters.get("solver") is not None:115            eval_kwargs["solver"] = eval_parameters["solver"]116 117        if eval_parameters.get("scorer") is not None:118            eval_kwargs["scorer"] = eval_parameters["scorer"]119 120        if self.log_dir is not None:121            eval_kwargs["log_dir"] = self.log_dir122 123        # Run evaluation124        logs = inspect_eval(task, model=model, **eval_kwargs)125 126        # Extract results from the first log127        if not logs:128            raise RuntimeError(129                "Inspect AI evaluation returned no logs. "130                "Check that the task and model arguments are valid."131            )132        log = logs[0]133        if log.status != "success":134            raise RuntimeError(135                f"Inspect AI evaluation failed with status: {log.status}"136            )137 138        return self._extract_scores(log)139 140    def _extract_scores(self, log: Any) -> Dict[str, Any]:141        """Parse an EvalLog's results into a flat score dictionary.142 143        Iterates over ``log.results.scores`` (a list of ``EvalScore``),144        flattening each scorer's ``metrics`` dict into a single output dict.145 146        Args:147            log: An ``inspect_ai`` ``EvalLog`` object.148 149        Returns:150            Dictionary mapping metric names to their values.151        """152        scores: Dict[str, Any] = {}153        if log.results is None:154            return scores155 156        for eval_score in log.results.scores:157            for metric_name, metric in eval_score.metrics.items():158                scores[metric_name] = metric.value159 160        return scores161