openenv/repl

1# Copyright (c) Meta Platforms, Inc. and affiliates.2# All rights reserved.3#4# This source code is licensed under the BSD-style license found in the5# LICENSE file in the root directory of this source tree.6 7"""Base class for evaluation harnesses."""8 9from abc import ABC, abstractmethod10from typing import Any, Dict11 12from openenv.core.evals.types import EvalConfig, EvalResult13 14 15class EvalHarness(ABC):16    """Abstract base class for evaluation harnesses.17 18    Subclasses implement run() to define evaluation logic.19    """20 21    @abstractmethod22    def run(23        self,24        harness_version: str,25        library_versions: Dict[str, str],26        dataset: str,27        eval_parameters: Dict[str, Any],28    ) -> Dict[str, Any]:29        """Run the evaluation and return scores.30 31        Args:32            harness_version: Version of the evaluation harness.33            library_versions: Versions of libraries used in the evaluation.34            dataset: Name of the dataset to evaluate on.35            eval_parameters: Parameters for the evaluation.36 37        Returns:38            Dictionary of scores from the evaluation.39        """40        raise NotImplementedError41 42    def run_from_config(self, config: EvalConfig) -> EvalResult:43        """Run evaluation from an EvalConfig and return an EvalResult.44 45        Args:46            config: Configuration for the evaluation.47 48        Returns:49            EvalResult containing the config and scores.50        """51        scores = self.run(52            harness_version=config.harness_version,53            library_versions=config.library_versions,54            dataset=config.dataset,55            eval_parameters=config.eval_parameters,56        )57        return EvalResult(config=config, scores=scores)58 59    @property60    def name(self) -> str:61        """Return the name of the harness (class name)."""62        return self.__class__.__name__63