codekingpro/portable-devtools

1"""This module contains the StringEvaluator class."""2 3import uuid4from typing import Callable, Optional5 6from pydantic import BaseModel7 8from langsmith.evaluation.evaluator import EvaluationResult, RunEvaluator9from langsmith.schemas import Example, Run10 11 12class StringEvaluator(RunEvaluator, BaseModel):13    """Grades the run's string input, output, and optional answer.14 15    .. deprecated:: 0.5.016 17       StringEvaluator is deprecated. Use openevals instead: https://github.com/langchain-ai/openevals18    """19 20    evaluation_name: Optional[str] = None21    """The name evaluation, such as `'Accuracy'` or `'Salience'`."""22    input_key: str = "input"23    """The key in the run inputs to extract the input string."""24    prediction_key: str = "output"25    """The key in the run outputs to extra the prediction string."""26    answer_key: Optional[str] = "output"27    """The key in the example outputs the answer string."""28    grading_function: Callable[[str, str, Optional[str]], dict]29    """Function that grades the run output against the example output."""30 31    def evaluate_run(32        self,33        run: Run,34        example: Optional[Example] = None,35        evaluator_run_id: Optional[uuid.UUID] = None,36    ) -> EvaluationResult:37        """Evaluate a single run."""38        if run.outputs is None:39            raise ValueError("Run outputs cannot be None.")40        if not example or example.outputs is None or self.answer_key is None:41            answer = None42        else:43            answer = example.outputs.get(self.answer_key)44        run_input = run.inputs[self.input_key]45        run_output = run.outputs[self.prediction_key]46        grading_results = self.grading_function(run_input, run_output, answer)47        return EvaluationResult(**{"key": self.evaluation_name, **grading_results})48