Source code for langchain.smith.evaluation.config
"""Configuration for run evaluators."""
from typing import Any, Dict, List, Optional, Union
from langsmith import RunEvaluator
from langchain.evaluation.criteria.eval_chain import CRITERIA_TYPE
from langchain.evaluation.embedding_distance.base import (
    EmbeddingDistance as EmbeddingDistanceEnum,
)
from langchain.evaluation.schema import EvaluatorType, StringEvaluator
from langchain.evaluation.string_distance.base import (
    StringDistance as StringDistanceEnum,
)
from langchain.pydantic_v1 import BaseModel, Field
from langchain.schema.embeddings import Embeddings
from langchain.schema.language_model import BaseLanguageModel
from langchain.schema.prompt_template import BasePromptTemplate
[docs]class EvalConfig(BaseModel):
    """Configuration for a given run evaluator.
    Parameters
    ----------
    evaluator_type : EvaluatorType
        The type of evaluator to use.
    Methods
    -------
    get_kwargs()
        Get the keyword arguments for the evaluator configuration.
    """
    evaluator_type: EvaluatorType
[docs]    def get_kwargs(self) -> Dict[str, Any]:
        """Get the keyword arguments for the load_evaluator call.
        Returns
        -------
        Dict[str, Any]
            The keyword arguments for the load_evaluator call.
        """
        kwargs = {}
        for field, val in self:
            if field == "evaluator_type":
                continue
            elif val is None:
                continue
            kwargs[field] = val
        return kwargs
[docs]class RunEvalConfig(BaseModel):
    """Configuration for a run evaluation.
    Parameters
    ----------
    evaluators : List[Union[EvaluatorType, EvalConfig]]
        Configurations for which evaluators to apply to the dataset run.