openenv/echo_env

1# SPDX-License-Identifier: BSD-3-Clause2 3"""Base class for evaluation harnesses."""4 5from abc import ABC, abstractmethod6from typing import Any, Dict7 8from openenv.core.evals.types import EvalConfig, EvalResult9 10 11class EvalHarness(ABC):12    """Abstract base class for evaluation harnesses.13 14    Subclasses implement run() to define evaluation logic.15    """16 17    @abstractmethod18    def run(19        self,20        harness_version: str,21        library_versions: Dict[str, str],22        dataset: str,23        eval_parameters: Dict[str, Any],24    ) -> Dict[str, Any]:25        """Run the evaluation and return scores.26 27        Args:28            harness_version: Version of the evaluation harness.29            library_versions: Versions of libraries used in the evaluation.30            dataset: Name of the dataset to evaluate on.31            eval_parameters: Parameters for the evaluation.32 33        Returns:34            Dictionary of scores from the evaluation.35        """36        raise NotImplementedError37 38    def run_from_config(self, config: EvalConfig) -> EvalResult:39        """Run evaluation from an EvalConfig and return an EvalResult.40 41        Args:42            config: Configuration for the evaluation.43 44        Returns:45            EvalResult containing the config and scores.46        """47        scores = self.run(48            harness_version=config.harness_version,49            library_versions=config.library_versions,50            dataset=config.dataset,51            eval_parameters=config.eval_parameters,52        )53        return EvalResult(config=config, scores=scores)54 55    @property56    def name(self) -> str:57        """Return the name of the harness (class name)."""58        return self.__class__.__name__59