Source code for langchain.evaluation.parsing.base
"""Evaluators for parsing strings."""
from operator import eq
from typing import Any, Callable, Optional, Union, cast
from langchain.evaluation.schema import StringEvaluator
from langchain.output_parsers.json import parse_json_markdown
[docs]class JsonValidityEvaluator(StringEvaluator):
    """Evaluates whether the prediction is valid JSON.
    This evaluator checks if the prediction is a valid JSON string. It does not
        require any input or reference.
    Attributes:
        requires_input (bool): Whether this evaluator requires an input
            string. Always False.
        requires_reference (bool): Whether this evaluator requires a
            reference string. Always False.
        evaluation_name (str): The name of the evaluation metric.
            Always "json".
    Examples:
        >>> evaluator = JsonValidityEvaluator()
        >>> prediction = '{"name": "John", "age": 30, "city": "New York"}'
        >>> evaluator.evaluate(prediction)
        {'score': 1}
        >>> prediction = '{"name": "John", "age": 30, "city": "New York",}'
        >>> evaluator.evaluate(prediction)
        {'score': 0, 'reasoning': 'Expecting property name enclosed in double quotes'}
    """
[docs]    def __init__(self, **kwargs: Any) -> None:
        super().__init__()
    @property
    def requires_input(self) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        return False
    @property
    def evaluation_name(self) -> str:
        return "json_validity"
    def _evaluate_strings(
        self,
        prediction: str,
        input: Optional[str] = None,