raise ValueError("Processor name is not defined, aborting!")
        output_path = gcs_output_path if gcs_output_path else self._gcs_output_path
        if output_path is None:
            raise ValueError("An output path on GCS should be provided!")
        operations = []
        for batch in batch_iterate(size=batch_size, iterable=blobs):
            documents = []
            for blob in batch:
                gcs_document = documentai.GcsDocument(
                    gcs_uri=blob.path, mime_type="application/pdf"
                )
                documents.append(gcs_document)
            gcs_documents = documentai.GcsDocuments(documents=documents)
            input_config = documentai.BatchDocumentsInputConfig(
                gcs_documents=gcs_documents
            )
            gcs_output_config = documentai.DocumentOutputConfig.GcsOutputConfig(
                gcs_uri=output_path, field_mask=None
            )
            output_config = documentai.DocumentOutputConfig(
                gcs_output_config=gcs_output_config
            )
            if enable_native_pdf_parsing:
                process_options = ProcessOptions(
                    ocr_config=OcrConfig(
                        enable_native_pdf_parsing=enable_native_pdf_parsing
                    )
                )
            else:
                process_options = ProcessOptions()
            request = documentai.BatchProcessRequest(
                name=self._processor_name,
                input_documents=input_config,
                document_output_config=output_config,
                process_options=process_options,
            )
            operations.append(self._client.batch_process_documents(request))
        return operations
[docs]    def get_results(self, operations: List["Operation"]) -> List[DocAIParsingResults]:
        try:
            from google.cloud.documentai_v1 import BatchProcessMetadata
        except ImportError:
            raise ImportError(


except ImportError:
            raise ImportError(
                "documentai package not found, please install it with"
                " `pip install google-cloud-documentai`"
            )
        results = []
        for op in operations:
            if isinstance(op.metadata, BatchProcessMetadata):
                metadata = op.metadata
            else:
                metadata = BatchProcessMetadata.deserialize(op.metadata.value)
            for status in metadata.individual_process_statuses:
                source = status.input_gcs_source
                output = status.output_gcs_destination
                results.append(
                    DocAIParsingResults(source_path=source, parsed_path=output)
                )
        return results


Source code for langchain.document_loaders.parsers.language.code_segmenter
from abc import ABC, abstractmethod
from typing import List
[docs]class CodeSegmenter(ABC):
    """Abstract class for the code segmenter."""
[docs]    def __init__(self, code: str):
        self.code = code
[docs]    def is_valid(self) -> bool:
        return True
[docs]    @abstractmethod
    def simplify_code(self) -> str:
        raise NotImplementedError()  # pragma: no cover
[docs]    @abstractmethod
    def extract_functions_classes(self) -> List[str]:
        raise NotImplementedError()  # pragma: no cover


Source code for langchain.document_loaders.parsers.language.language_parser
from typing import Any, Dict, Iterator, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseBlobParser
from langchain.document_loaders.blob_loaders import Blob
from langchain.document_loaders.parsers.language.javascript import JavaScriptSegmenter
from langchain.document_loaders.parsers.language.python import PythonSegmenter
from langchain.text_splitter import Language
LANGUAGE_EXTENSIONS: Dict[str, str] = {
    "py": Language.PYTHON,
    "js": Language.JS,
}
LANGUAGE_SEGMENTERS: Dict[str, Any] = {
    Language.PYTHON: PythonSegmenter,
    Language.JS: JavaScriptSegmenter,
}
[docs]class LanguageParser(BaseBlobParser):
    """Parse using the respective programming language syntax.
    Each top-level function and class in the code is loaded into separate documents.
    Furthermore, an extra document is generated, containing the remaining top-level code
    that excludes the already segmented functions and classes.
    This approach can potentially improve the accuracy of QA models over source code.
    Currently, the supported languages for code parsing are Python and JavaScript.
    The language used for parsing can be configured, along with the minimum number of
    lines required to activate the splitting based on syntax.
    Examples:
       .. code-block:: python
            from langchain.text_splitter.Language
            from langchain.document_loaders.generic import GenericLoader
            from langchain.document_loaders.parsers import LanguageParser
            loader = GenericLoader.from_filesystem(
                "./code",
                glob="**/*",
                suffixes=[".py", ".js"],
                parser=LanguageParser()
            )
            docs = loader.load()
        Example instantiations to manually select the language:
        .. code-block:: python


Example instantiations to manually select the language:
        .. code-block:: python
            from langchain.text_splitter import Language
            loader = GenericLoader.from_filesystem(
                "./code",
                glob="**/*",
                suffixes=[".py"],
                parser=LanguageParser(language=Language.PYTHON)
            )
        Example instantiations to set number of lines threshold:
        .. code-block:: python
            loader = GenericLoader.from_filesystem(
                "./code",
                glob="**/*",
                suffixes=[".py"],
                parser=LanguageParser(parser_threshold=200)
            )
    """
[docs]    def __init__(self, language: Optional[Language] = None, parser_threshold: int = 0):
        """
        Language parser that split code using the respective language syntax.
        Args:
            language: If None (default), it will try to infer language from source.
            parser_threshold: Minimum lines needed to activate parsing (0 by default).
        """
        self.language = language
        self.parser_threshold = parser_threshold
[docs]    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
        code = blob.as_string()
        language = self.language or (
            LANGUAGE_EXTENSIONS.get(blob.source.rsplit(".", 1)[-1])
            if isinstance(blob.source, str)
            else None
        )
        if language is None:
            yield Document(
                page_content=code,
                metadata={
                    "source": blob.source,
                },
            )
            return
        if self.parser_threshold >= len(code.splitlines()):
            yield Document(
                page_content=code,
                metadata={
                    "source": blob.source,
                    "language": language,
                },
            )
            return


"language": language,
                },
            )
            return
        self.Segmenter = LANGUAGE_SEGMENTERS[language]
        segmenter = self.Segmenter(blob.as_string())
        if not segmenter.is_valid():
            yield Document(
                page_content=code,
                metadata={
                    "source": blob.source,
                },
            )
            return
        for functions_classes in segmenter.extract_functions_classes():
            yield Document(
                page_content=functions_classes,
                metadata={
                    "source": blob.source,
                    "content_type": "functions_classes",
                    "language": language,
                },
            )
        yield Document(
            page_content=segmenter.simplify_code(),
            metadata={
                "source": blob.source,
                "content_type": "simplified_code",
                "language": language,
            },
        )


Source code for langchain.document_loaders.parsers.language.python
import ast
from typing import Any, List
from langchain.document_loaders.parsers.language.code_segmenter import CodeSegmenter
[docs]class PythonSegmenter(CodeSegmenter):
    """Code segmenter for `Python`."""
[docs]    def __init__(self, code: str):
        super().__init__(code)
        self.source_lines = self.code.splitlines()
[docs]    def is_valid(self) -> bool:
        try:
            ast.parse(self.code)
            return True
        except SyntaxError:
            return False
    def _extract_code(self, node: Any) -> str:
        start = node.lineno - 1
        end = node.end_lineno
        return "\n".join(self.source_lines[start:end])
[docs]    def extract_functions_classes(self) -> List[str]:
        tree = ast.parse(self.code)
        functions_classes = []
        for node in ast.iter_child_nodes(tree):
            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
                functions_classes.append(self._extract_code(node))
        return functions_classes
[docs]    def simplify_code(self) -> str:
        tree = ast.parse(self.code)
        simplified_lines = self.source_lines[:]
        for node in ast.iter_child_nodes(tree):
            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
                start = node.lineno - 1
                simplified_lines[start] = f"# Code for: {simplified_lines[start]}"
                assert isinstance(node.end_lineno, int)
                for line_num in range(start + 1, node.end_lineno):
                    simplified_lines[line_num] = None  # type: ignore


simplified_lines[line_num] = None  # type: ignore
        return "\n".join(line for line in simplified_lines if line is not None)


Source code for langchain.document_loaders.parsers.language.javascript
from typing import Any, List
from langchain.document_loaders.parsers.language.code_segmenter import CodeSegmenter
[docs]class JavaScriptSegmenter(CodeSegmenter):
    """Code segmenter for JavaScript."""
[docs]    def __init__(self, code: str):
        super().__init__(code)
        self.source_lines = self.code.splitlines()
        try:
            import esprima  # noqa: F401
        except ImportError:
            raise ImportError(
                "Could not import esprima Python package. "
                "Please install it with `pip install esprima`."
            )
[docs]    def is_valid(self) -> bool:
        import esprima
        try:
            esprima.parseScript(self.code)
            return True
        except esprima.Error:
            return False
    def _extract_code(self, node: Any) -> str:
        start = node.loc.start.line - 1
        end = node.loc.end.line
        return "\n".join(self.source_lines[start:end])
[docs]    def extract_functions_classes(self) -> List[str]:
        import esprima
        tree = esprima.parseScript(self.code, loc=True)
        functions_classes = []
        for node in tree.body:
            if isinstance(
                node,
                (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration),
            ):
                functions_classes.append(self._extract_code(node))
        return functions_classes
[docs]    def simplify_code(self) -> str:
        import esprima
        tree = esprima.parseScript(self.code, loc=True)
        simplified_lines = self.source_lines[:]
        for node in tree.body:
            if isinstance(
                node,


for node in tree.body:
            if isinstance(
                node,
                (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration),
            ):
                start = node.loc.start.line - 1
                simplified_lines[start] = f"// Code for: {simplified_lines[start]}"
                for line_num in range(start + 1, node.loc.end.line):
                    simplified_lines[line_num] = None  # type: ignore
        return "\n".join(line for line in simplified_lines if line is not None)


Source code for langchain.document_loaders.parsers.html.bs4
"""Loader that uses bs4 to load HTML files, enriching metadata with page title."""
import logging
from typing import Any, Dict, Iterator, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseBlobParser
from langchain.document_loaders.blob_loaders import Blob
logger = logging.getLogger(__name__)
[docs]class BS4HTMLParser(BaseBlobParser):
    """Pparse HTML files using `Beautiful Soup`."""
[docs]    def __init__(
        self,
        *,
        features: str = "lxml",
        get_text_separator: str = "",
        **kwargs: Any,
    ) -> None:
        """Initialize a bs4 based HTML parser."""
        try:
            import bs4  # noqa:F401
        except ImportError:
            raise ImportError(
                "beautifulsoup4 package not found, please install it with "
                "`pip install beautifulsoup4`"
            )
        self.bs_kwargs = {"features": features, **kwargs}
        self.get_text_separator = get_text_separator
[docs]    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
        """Load HTML document into document objects."""
        from bs4 import BeautifulSoup
        with blob.as_bytes_io() as f:
            soup = BeautifulSoup(f, **self.bs_kwargs)
        text = soup.get_text(self.get_text_separator)
        if soup.title:
            title = str(soup.title.string)
        else:
            title = ""
        metadata: Dict[str, Union[str, None]] = {
            "source": blob.source,
            "title": title,
        }
        yield Document(page_content=text, metadata=metadata)


Source code for langchain.document_loaders.blob_loaders.file_system
"""Use to load blobs from the local file system."""
from pathlib import Path
from typing import Callable, Iterable, Iterator, Optional, Sequence, TypeVar, Union
from langchain.document_loaders.blob_loaders.schema import Blob, BlobLoader
T = TypeVar("T")
def _make_iterator(
    length_func: Callable[[], int], show_progress: bool = False
) -> Callable[[Iterable[T]], Iterator[T]]:
    """Create a function that optionally wraps an iterable in tqdm."""
    if show_progress:
        try:
            from tqdm.auto import tqdm
        except ImportError:
            raise ImportError(
                "You must install tqdm to use show_progress=True."
                "You can install tqdm with `pip install tqdm`."
            )
        # Make sure to provide `total` here so that tqdm can show
        # a progress bar that takes into account the total number of files.
        def _with_tqdm(iterable: Iterable[T]) -> Iterator[T]:
            """Wrap an iterable in a tqdm progress bar."""
            return tqdm(iterable, total=length_func())
        iterator = _with_tqdm
    else:
        iterator = iter  # type: ignore
    return iterator
# PUBLIC API
[docs]class FileSystemBlobLoader(BlobLoader):
    """Load blobs in the local file system.
    Example:
    .. code-block:: python
        from langchain.document_loaders.blob_loaders import FileSystemBlobLoader
        loader = FileSystemBlobLoader("/path/to/directory")
        for blob in loader.yield_blobs():
            print(blob)
    """
[docs]    def __init__(
        self,
        path: Union[str, Path],
        *,
        glob: str = "**/[!.]*",


*,
        glob: str = "**/[!.]*",
        exclude: Sequence[str] = (),
        suffixes: Optional[Sequence[str]] = None,
        show_progress: bool = False,
    ) -> None:
        """Initialize with a path to directory and how to glob over it.
        Args:
            path: Path to directory to load from
            glob: Glob pattern relative to the specified path
                  by default set to pick up all non-hidden files
            exclude: patterns to exclude from results, use glob syntax
            suffixes: Provide to keep only files with these suffixes
                      Useful when wanting to keep files with different suffixes
                      Suffixes must include the dot, e.g. ".txt"
            show_progress: If true, will show a progress bar as the files are loaded.
                           This forces an iteration through all matching files
                           to count them prior to loading them.
        Examples:
            .. code-block:: python
                # Recursively load all text files in a directory.
                loader = FileSystemBlobLoader("/path/to/directory", glob="**/*.txt")
                # Recursively load all non-hidden files in a directory.
                loader = FileSystemBlobLoader("/path/to/directory", glob="**/[!.]*")
                # Load all files in a directory without recursion.
                loader = FileSystemBlobLoader("/path/to/directory", glob="*")
                # Recursively load all files in a directory, except for py or pyc files.
                loader = FileSystemBlobLoader(
                    "/path/to/directory",
                    glob="**/*.txt",
                    exclude=["**/*.py", "**/*.pyc"]
                )
        """
        if isinstance(path, Path):
            _path = path
        elif isinstance(path, str):
            _path = Path(path)


elif isinstance(path, str):
            _path = Path(path)
        else:
            raise TypeError(f"Expected str or Path, got {type(path)}")
        self.path = _path.expanduser()  # Expand user to handle ~
        self.glob = glob
        self.suffixes = set(suffixes or [])
        self.show_progress = show_progress
        self.exclude = exclude
[docs]    def yield_blobs(
        self,
    ) -> Iterable[Blob]:
        """Yield blobs that match the requested pattern."""
        iterator = _make_iterator(
            length_func=self.count_matching_files, show_progress=self.show_progress
        )
        for path in iterator(self._yield_paths()):
            yield Blob.from_path(path)
    def _yield_paths(self) -> Iterable[Path]:
        """Yield paths that match the requested pattern."""
        paths = self.path.glob(self.glob)
        for path in paths:
            if self.exclude:
                if any(path.match(glob) for glob in self.exclude):
                    continue
            if path.is_file():
                if self.suffixes and path.suffix not in self.suffixes:
                    continue
                yield path
[docs]    def count_matching_files(self) -> int:
        """Count files that match the pattern without loading them."""
        # Carry out a full iteration to count the files without
        # materializing anything expensive in memory.
        num = 0
        for _ in self._yield_paths():
            num += 1
        return num


Source code for langchain.document_loaders.blob_loaders.schema
"""Schema for Blobs and Blob Loaders.
The goal is to facilitate decoupling of content loading from content parsing code.
In addition, content loading code should provide a lazy loading interface by default.
"""
from __future__ import annotations
import contextlib
import mimetypes
from abc import ABC, abstractmethod
from io import BufferedReader, BytesIO
from pathlib import PurePath
from typing import Any, Generator, Iterable, Mapping, Optional, Union
from langchain.pydantic_v1 import BaseModel, root_validator
PathLike = Union[str, PurePath]
[docs]class Blob(BaseModel):
    """Blob represents raw data by either reference or value.
    Provides an interface to materialize the blob in different representations, and
    help to decouple the development of data loaders from the downstream parsing of
    the raw data.
    Inspired by: https://developer.mozilla.org/en-US/docs/Web/API/Blob
    """
    data: Union[bytes, str, None]  # Raw data
    mimetype: Optional[str] = None  # Not to be confused with a file extension
    encoding: str = "utf-8"  # Use utf-8 as default encoding, if decoding to string
    # Location where the original content was found
    # Represent location on the local file system
    # Useful for situations where downstream code assumes it must work with file paths
    # rather than in-memory content.
    path: Optional[PathLike] = None
    class Config:
        arbitrary_types_allowed = True
        frozen = True
    @property
    def source(self) -> Optional[str]:
        """The source location of the blob as string if known otherwise none."""
        return str(self.path) if self.path else None
    @root_validator(pre=True)


return str(self.path) if self.path else None
    @root_validator(pre=True)
    def check_blob_is_valid(cls, values: Mapping[str, Any]) -> Mapping[str, Any]:
        """Verify that either data or path is provided."""
        if "data" not in values and "path" not in values:
            raise ValueError("Either data or path must be provided")
        return values
[docs]    def as_string(self) -> str:
        """Read data as a string."""
        if self.data is None and self.path:
            with open(str(self.path), "r", encoding=self.encoding) as f:
                return f.read()
        elif isinstance(self.data, bytes):
            return self.data.decode(self.encoding)
        elif isinstance(self.data, str):
            return self.data
        else:
            raise ValueError(f"Unable to get string for blob {self}")
[docs]    def as_bytes(self) -> bytes:
        """Read data as bytes."""
        if isinstance(self.data, bytes):
            return self.data
        elif isinstance(self.data, str):
            return self.data.encode(self.encoding)
        elif self.data is None and self.path:
            with open(str(self.path), "rb") as f:
                return f.read()
        else:
            raise ValueError(f"Unable to get bytes for blob {self}")
[docs]    @contextlib.contextmanager
    def as_bytes_io(self) -> Generator[Union[BytesIO, BufferedReader], None, None]:
        """Read data as a byte stream."""
        if isinstance(self.data, bytes):
            yield BytesIO(self.data)
        elif self.data is None and self.path:
            with open(str(self.path), "rb") as f:
                yield f
        else:


yield f
        else:
            raise NotImplementedError(f"Unable to convert blob {self}")
[docs]    @classmethod
    def from_path(
        cls,
        path: PathLike,
        *,
        encoding: str = "utf-8",
        mime_type: Optional[str] = None,
        guess_type: bool = True,
    ) -> Blob:
        """Load the blob from a path like object.
        Args:
            path: path like object to file to be read
            encoding: Encoding to use if decoding the bytes into a string
            mime_type: if provided, will be set as the mime-type of the data
            guess_type: If True, the mimetype will be guessed from the file extension,
                        if a mime-type was not provided
        Returns:
            Blob instance
        """
        if mime_type is None and guess_type:
            _mimetype = mimetypes.guess_type(path)[0] if guess_type else None
        else:
            _mimetype = mime_type
        # We do not load the data immediately, instead we treat the blob as a
        # reference to the underlying data.
        return cls(data=None, mimetype=_mimetype, encoding=encoding, path=path)
[docs]    @classmethod
    def from_data(
        cls,
        data: Union[str, bytes],
        *,
        encoding: str = "utf-8",
        mime_type: Optional[str] = None,
        path: Optional[str] = None,
    ) -> Blob:
        """Initialize the blob from in-memory data.
        Args:
            data: the in-memory data associated with the blob
            encoding: Encoding to use if decoding the bytes into a string
            mime_type: if provided, will be set as the mime-type of the data


mime_type: if provided, will be set as the mime-type of the data
            path: if provided, will be set as the source from which the data came
        Returns:
            Blob instance
        """
        return cls(data=data, mimetype=mime_type, encoding=encoding, path=path)
    def __repr__(self) -> str:
        """Define the blob representation."""
        str_repr = f"Blob {id(self)}"
        if self.source:
            str_repr += f" {self.source}"
        return str_repr
[docs]class BlobLoader(ABC):
    """Abstract interface for blob loaders implementation.
    Implementer should be able to load raw content from a storage system according
    to some criteria and return the raw content lazily as a stream of blobs.
    """
[docs]    @abstractmethod
    def yield_blobs(
        self,
    ) -> Iterable[Blob]:
        """A lazy loader for raw data represented by LangChain's Blob object.
        Returns:
            A generator over blobs
        """


Source code for langchain.document_loaders.blob_loaders.youtube_audio
from typing import Iterable, List
from langchain.document_loaders.blob_loaders import FileSystemBlobLoader
from langchain.document_loaders.blob_loaders.schema import Blob, BlobLoader
[docs]class YoutubeAudioLoader(BlobLoader):
    """Load YouTube urls as audio file(s)."""
[docs]    def __init__(self, urls: List[str], save_dir: str):
        if not isinstance(urls, list):
            raise TypeError("urls must be a list")
        self.urls = urls
        self.save_dir = save_dir
[docs]    def yield_blobs(self) -> Iterable[Blob]:
        """Yield audio blobs for each url."""
        try:
            import yt_dlp
        except ImportError:
            raise ImportError(
                "yt_dlp package not found, please install it with "
                "`pip install yt_dlp`"
            )
        # Use yt_dlp to download audio given a YouTube url
        ydl_opts = {
            "format": "m4a/bestaudio/best",
            "noplaylist": True,
            "outtmpl": self.save_dir + "/%(title)s.%(ext)s",
            "postprocessors": [
                {
                    "key": "FFmpegExtractAudio",
                    "preferredcodec": "m4a",
                }
            ],
        }
        for url in self.urls:
            # Download file
            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                ydl.download(url)
        # Yield the written blobs
        loader = FileSystemBlobLoader(self.save_dir, glob="*.m4a")
        for blob in loader.yield_blobs():
            yield blob


Source code for langchain.smith.evaluation.string_run_evaluator
"""Run evaluator wrapper for string evaluators."""
from __future__ import annotations
from abc import abstractmethod
from typing import Any, Dict, List, Optional
from langsmith import EvaluationResult, RunEvaluator
from langsmith.schemas import DataType, Example, Run
from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
)
from langchain.chains.base import Chain
from langchain.evaluation.schema import StringEvaluator
from langchain.load.dump import dumpd
from langchain.load.load import load
from langchain.load.serializable import Serializable
from langchain.schema import RUN_KEY, messages_from_dict
from langchain.schema.messages import BaseMessage, get_buffer_string
def _get_messages_from_run_dict(messages: List[dict]) -> List[BaseMessage]:
    if not messages:
        return []
    first_message = messages[0]
    if "lc" in first_message:
        return [load(dumpd(message)) for message in messages]
    else:
        return messages_from_dict(messages)
[docs]class StringRunMapper(Serializable):
    """Extract items to evaluate from the run object."""
    @property
    def output_keys(self) -> List[str]:
        """The keys to extract from the run."""
        return ["prediction", "input"]
[docs]    @abstractmethod
    def map(self, run: Run) -> Dict[str, str]:
        """Maps the Run to a dictionary."""
[docs]    def __call__(self, run: Run) -> Dict[str, str]:
        """Maps the Run to a dictionary."""
        if not run.outputs:
            raise ValueError(f"Run {run.id} has no outputs to evaluate.")
        return self.map(run)


return self.map(run)
[docs]class LLMStringRunMapper(StringRunMapper):
    """Extract items to evaluate from the run object."""
[docs]    def serialize_chat_messages(self, messages: List[Dict]) -> str:
        """Extract the input messages from the run."""
        if isinstance(messages, list) and messages:
            if isinstance(messages[0], dict):
                chat_messages = _get_messages_from_run_dict(messages)
            elif isinstance(messages[0], list):
                # Runs from Tracer have messages as a list of lists of dicts
                chat_messages = _get_messages_from_run_dict(messages[0])
            else:
                raise ValueError(f"Could not extract messages to evaluate {messages}")
            return get_buffer_string(chat_messages)
        raise ValueError(f"Could not extract messages to evaluate {messages}")
[docs]    def serialize_inputs(self, inputs: Dict) -> str:
        if "prompts" in inputs:  # Should we even accept this?
            input_ = "\n\n".join(inputs["prompts"])
        elif "prompt" in inputs:
            input_ = inputs["prompt"]
        elif "messages" in inputs:
            input_ = self.serialize_chat_messages(inputs["messages"])
        else:
            raise ValueError("LLM Run must have either messages or prompts as inputs.")
        return input_
[docs]    def serialize_outputs(self, outputs: Dict) -> str:
        if not outputs.get("generations"):
            raise ValueError("Cannot evaluate LLM Run without generations.")
        generations: List[Dict] = outputs["generations"]
        if not generations:
            raise ValueError("Cannot evaluate LLM run with empty generations.")
        first_generation: Dict = generations[0]
        if isinstance(first_generation, list):


first_generation: Dict = generations[0]
        if isinstance(first_generation, list):
            # Runs from Tracer have generations as a list of lists of dicts
            # Whereas Runs from the API have a list of dicts
            first_generation = first_generation[0]
        if "message" in first_generation:
            output_ = self.serialize_chat_messages([first_generation["message"]])
        else:
            output_ = first_generation["text"]
        return output_
[docs]    def map(self, run: Run) -> Dict[str, str]:
        """Maps the Run to a dictionary."""
        if run.run_type != "llm":
            raise ValueError("LLM RunMapper only supports LLM runs.")
        elif not run.outputs:
            if run.error:
                raise ValueError(
                    f"Cannot evaluate errored LLM run {run.id}: {run.error}"
                )
            else:
                raise ValueError(
                    f"Run {run.id} has no outputs. Cannot evaluate this run."
                )
        else:
            try:
                inputs = self.serialize_inputs(run.inputs)
            except Exception as e:
                raise ValueError(
                    f"Could not parse LM input from run inputs {run.inputs}"
                ) from e
            try:
                output_ = self.serialize_outputs(run.outputs)
            except Exception as e:
                raise ValueError(
                    f"Could not parse LM prediction from run outputs {run.outputs}"
                ) from e
            return {"input": inputs, "prediction": output_}
[docs]class ChainStringRunMapper(StringRunMapper):
    """Extract items to evaluate from the run object from a chain."""
    input_key: Optional[str] = None
    """The key from the model Run's inputs to use as the eval input.


"""The key from the model Run's inputs to use as the eval input.
    If not provided, will use the only input key or raise an
    error if there are multiple."""
    prediction_key: Optional[str] = None
    """The key from the model Run's outputs to use as the eval prediction.
    If not provided, will use the only output key or raise an error
    if there are multiple."""
    def _get_key(self, source: Dict, key: Optional[str], which: str) -> str:
        if key is not None:
            return source[key]
        elif len(source) == 1:
            return next(iter(source.values()))
        else:
            raise ValueError(
                f"Could not map run {which} with multiple keys: "
                f"{source}\nPlease manually specify a {which}_key"
            )
[docs]    def map(self, run: Run) -> Dict[str, str]:
        """Maps the Run to a dictionary."""
        if not run.outputs:
            raise ValueError(
                f"Run with ID {run.id} lacks outputs required for evaluation."
                " Ensure the Run has valid outputs."
            )
        if self.input_key is not None and self.input_key not in run.inputs:
            raise ValueError(
                f"Run with ID {run.id} is missing the expected input key"
                f" '{self.input_key}'.\nAvailable input keys in this Run"
                f"  are: {run.inputs.keys()}.\nAdjust the evaluator's"
                f" input_key or ensure your input data includes key"
                f" '{self.input_key}'."
            )
        elif self.prediction_key is not None and self.prediction_key not in run.outputs:
            available_keys = ", ".join(run.outputs.keys())


available_keys = ", ".join(run.outputs.keys())
            raise ValueError(
                f"Run with ID {run.id} doesn't have the expected prediction key"
                f" '{self.prediction_key}'. Available prediction keys in this Run are:"
                f" {available_keys}. Adjust the evaluator's prediction_key or"
                " ensure the Run object's outputs the expected key."
            )
        else:
            input_ = self._get_key(run.inputs, self.input_key, "input")
            prediction = self._get_key(run.outputs, self.prediction_key, "prediction")
            return {
                "input": input_,
                "prediction": prediction,
            }
[docs]class ToolStringRunMapper(StringRunMapper):
    """Map an input to the tool."""
[docs]    def map(self, run: Run) -> Dict[str, str]:
        if not run.outputs:
            raise ValueError(f"Run {run.id} has no outputs to evaluate.")
        return {"input": run.inputs["input"], "prediction": run.outputs["output"]}
[docs]class StringExampleMapper(Serializable):
    """Map an example, or row in the dataset, to the inputs of an evaluation."""
    reference_key: Optional[str] = None
    @property
    def output_keys(self) -> List[str]:
        """The keys to extract from the run."""
        return ["reference"]
[docs]    def serialize_chat_messages(self, messages: List[Dict]) -> str:
        """Extract the input messages from the run."""
        chat_messages = _get_messages_from_run_dict(messages)
        return get_buffer_string(chat_messages)
[docs]    def map(self, example: Example) -> Dict[str, str]:
        """Maps the Example, or dataset row to a dictionary."""


"""Maps the Example, or dataset row to a dictionary."""
        if not example.outputs:
            raise ValueError(
                f"Example {example.id} has no outputs to use as a reference."
            )
        if self.reference_key is None:
            if len(example.outputs) > 1:
                raise ValueError(
                    f"Example {example.id} has multiple outputs, so you must"
                    " specify a reference_key."
                )
            else:
                output = list(example.outputs.values())[0]
        elif self.reference_key not in example.outputs:
            raise ValueError(
                f"Example {example.id} does not have reference key"
                f" {self.reference_key}."
            )
        else:
            output = example.outputs[self.reference_key]
        return {
            "reference": self.serialize_chat_messages([output])
            if isinstance(output, dict) and output.get("type") and output.get("data")
            else str(output)
        }
[docs]    def __call__(self, example: Example) -> Dict[str, str]:
        """Maps the Run and Example to a dictionary."""
        if not example.outputs:
            raise ValueError(
                f"Example {example.id} has no outputs to use as areference label."
            )
        return self.map(example)
[docs]class StringRunEvaluatorChain(Chain, RunEvaluator):
    """Evaluate Run and optional examples."""
    run_mapper: StringRunMapper
    """Maps the Run to a dictionary with 'input' and 'prediction' strings."""
    example_mapper: Optional[StringExampleMapper] = None
    """Maps the Example (dataset row) to a dictionary
    with a 'reference' string."""
    name: str
    """The name of the evaluation metric."""
    string_evaluator: StringEvaluator


"""The name of the evaluation metric."""
    string_evaluator: StringEvaluator
    """The evaluation chain."""
    @property
    def input_keys(self) -> List[str]:
        return ["run", "example"]
    @property
    def output_keys(self) -> List[str]:
        return ["feedback"]
    def _prepare_input(self, inputs: Dict[str, Any]) -> Dict[str, str]:
        run: Run = inputs["run"]
        example: Optional[Example] = inputs.get("example")
        evaluate_strings_inputs = self.run_mapper(run)
        if not self.string_evaluator.requires_input:
            # Hide warning about unused input
            evaluate_strings_inputs.pop("input", None)
        if example and self.example_mapper and self.string_evaluator.requires_reference:
            evaluate_strings_inputs.update(self.example_mapper(example))
        elif self.string_evaluator.requires_reference:
            raise ValueError(
                f"Evaluator {self.name} requires an reference"
                " example from the dataset,"
                f" but none was provided for run {run.id}."
            )
        return evaluate_strings_inputs
    def _prepare_output(self, output: Dict[str, Any]) -> Dict[str, Any]:
        evaluation_result = EvaluationResult(
            key=self.name, comment=output.get("reasoning"), **output
        )
        if RUN_KEY in output:
            # TODO: Not currently surfaced. Update
            evaluation_result.evaluator_info[RUN_KEY] = output[RUN_KEY]
        return {"feedback": evaluation_result}
    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Call the evaluation chain."""
        evaluate_strings_inputs = self._prepare_input(inputs)


"""Call the evaluation chain."""
        evaluate_strings_inputs = self._prepare_input(inputs)
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        callbacks = _run_manager.get_child()
        chain_output = self.string_evaluator.evaluate_strings(
            **evaluate_strings_inputs,
            callbacks=callbacks,
            include_run_info=True,
        )
        return self._prepare_output(chain_output)
    async def _acall(
        self,
        inputs: Dict[str, str],
        run_manager: AsyncCallbackManagerForChainRun | None = None,
    ) -> Dict[str, Any]:
        """Call the evaluation chain."""
        evaluate_strings_inputs = self._prepare_input(inputs)
        _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
        callbacks = _run_manager.get_child()
        chain_output = await self.string_evaluator.aevaluate_strings(
            **evaluate_strings_inputs,
            callbacks=callbacks,
            include_run_info=True,
        )
        return self._prepare_output(chain_output)
    def _prepare_evaluator_output(self, output: Dict[str, Any]) -> EvaluationResult:
        feedback: EvaluationResult = output["feedback"]
        if RUN_KEY not in feedback.evaluator_info:
            feedback.evaluator_info[RUN_KEY] = output[RUN_KEY]
        return feedback
[docs]    def evaluate_run(
        self, run: Run, example: Optional[Example] = None
    ) -> EvaluationResult:
        """Evaluate an example."""
        try:
            result = self({"run": run, "example": example}, include_run_info=True)
            return self._prepare_evaluator_output(result)
        except Exception as e:
            return EvaluationResult(
                key=self.string_evaluator.evaluation_name,


return EvaluationResult(
                key=self.string_evaluator.evaluation_name,
                comment=f"Error evaluating run {run.id}: {e}",
                # TODO: Add run ID once we can declare it via callbacks
            )
[docs]    async def aevaluate_run(
        self, run: Run, example: Optional[Example] = None
    ) -> EvaluationResult:
        """Evaluate an example."""
        try:
            result = await self.acall(
                {"run": run, "example": example}, include_run_info=True
            )
            return self._prepare_evaluator_output(result)
        except Exception as e:
            return EvaluationResult(
                key=self.string_evaluator.evaluation_name,
                comment=f"Error evaluating run {run.id}: {e}",
            )
[docs]    @classmethod
    def from_run_and_data_type(
        cls,
        evaluator: StringEvaluator,
        run_type: str,
        data_type: DataType,
        input_key: Optional[str] = None,
        prediction_key: Optional[str] = None,
        reference_key: Optional[str] = None,
        tags: Optional[List[str]] = None,
    ) -> StringRunEvaluatorChain:
        """
        Create a StringRunEvaluatorChain from an evaluator and the run and dataset types.
        This method provides an easy way to instantiate a StringRunEvaluatorChain, by
        taking an evaluator and information about the type of run and the data.
        The method supports LLM and chain runs.
        Args:
            evaluator (StringEvaluator): The string evaluator to use.
            run_type (str): The type of run being evaluated.
                Supported types are LLM and Chain.
            data_type (DataType): The type of dataset used in the run.


data_type (DataType): The type of dataset used in the run.
            input_key (str, optional): The key used to map the input from the run.
            prediction_key (str, optional): The key used to map the prediction from the run.
            reference_key (str, optional): The key used to map the reference from the dataset.
            tags (List[str], optional): List of tags to attach to the evaluation chain.
        Returns:
            StringRunEvaluatorChain: The instantiated evaluation chain.
        Raises:
            ValueError: If the run type is not supported, or if the evaluator requires a
                reference from the dataset but the reference key is not provided.
        """  # noqa: E501
        # Configure how run inputs/predictions are passed to the evaluator
        if run_type == "llm":
            run_mapper: StringRunMapper = LLMStringRunMapper()
        elif run_type == "chain":
            run_mapper = ChainStringRunMapper(
                input_key=input_key, prediction_key=prediction_key
            )
        else:
            raise ValueError(
                f"Unsupported run type {run_type}. Expected one of 'llm' or 'chain'."
            )
        # Configure how example rows are fed as a reference string to the evaluator
        if (
            reference_key is not None
            or data_type in (DataType.llm, DataType.chat)
            or evaluator.requires_reference
        ):
            example_mapper = StringExampleMapper(reference_key=reference_key)
        elif evaluator.requires_reference:
            raise ValueError(
                f"Evaluator {evaluator.evaluation_name} requires a reference"
                " example from the dataset. Please specify the reference key from"
                " amongst the dataset outputs keys."
            )
        else:
            example_mapper = None
        return cls(


)
        else:
            example_mapper = None
        return cls(
            name=evaluator.evaluation_name,
            run_mapper=run_mapper,
            example_mapper=example_mapper,
            string_evaluator=evaluator,
            tags=tags,
        )


Source code for langchain.smith.evaluation.name_generation
import random
adjectives = [
    "abandoned",
    "aching",
    "advanced",
    "ample",
    "artistic",
    "back",
    "best",
    "bold",
    "brief",
    "clear",
    "cold",
    "complicated",
    "cooked",
    "crazy",
    "crushing",
    "damp",
    "dear",
    "definite",
    "dependable",
    "diligent",
    "drab",
    "earnest",
    "elderly",
    "enchanted",
    "essential",
    "excellent",
    "extraneous",
    "fixed",
    "flowery",
    "formal",
    "fresh",
    "frosty",
    "giving",
    "glossy",
    "healthy",
    "helpful",
    "impressionable",
    "kind",
    "large",
    "left",
    "long",
    "loyal",
    "mealy",
    "memorable",
    "monthly",
    "new",
    "notable",
    "only",
    "ordinary",
    "passionate",
    "perfect",
    "pertinent",
    "proper",
    "puzzled",
    "reflecting",
    "respectful",
    "roasted",
    "scholarly",
    "shiny",
    "slight",
    "sparkling",
    "spotless",
    "stupendous",
    "sunny",
    "tart",
    "terrific",


"sunny",
    "tart",
    "terrific",
    "timely",
    "unique",
    "upbeat",
    "vacant",
    "virtual",
    "warm",
    "weary",
    "whispered",
    "worthwhile",
    "yellow",
]
nouns = [
    "account",
    "acknowledgment",
    "address",
    "advertising",
    "airplane",
    "animal",
    "appointment",
    "arrival",
    "artist",
    "attachment",
    "attitude",
    "availability",
    "backpack",
    "bag",
    "balance",
    "bass",
    "bean",
    "beauty",
    "bibliography",
    "bill",
    "bite",
    "blossom",
    "boat",
    "book",
    "box",
    "boy",
    "bread",
    "bridge",
    "broccoli",
    "building",
    "butter",
    "button",
    "cabbage",
    "cake",
    "camera",
    "camp",
    "candle",
    "candy",
    "canvas",
    "car",
    "card",
    "carrot",
    "cart",
    "case",
    "cat",
    "chain",
    "chair",
    "chalk",
    "chance",
    "change",
    "channel",
    "character",
    "charge",
    "charm",
    "chart",
    "check",
    "cheek",
    "cheese",
    "chef",


"cheek",
    "cheese",
    "chef",
    "cherry",
    "chicken",
    "child",
    "church",
    "circle",
    "class",
    "clay",
    "click",
    "clock",
    "cloth",
    "cloud",
    "clove",
    "club",
    "coach",
    "coal",
    "coast",
    "coat",
    "cod",
    "coffee",
    "collar",
    "color",
    "comb",
    "comfort",
    "comic",
    "committee",
    "community",
    "company",
    "comparison",
    "competition",
    "condition",
    "connection",
    "control",
    "cook",
    "copper",
    "copy",
    "corn",
    "cough",
    "country",
    "cover",
    "crate",
    "crayon",
    "cream",
    "creator",
    "crew",
    "crown",
    "current",
    "curtain",
    "curve",
    "cushion",
    "dad",
    "daughter",
    "day",
    "death",
    "debt",
    "decision",
    "deer",
    "degree",
    "design",
    "desire",
    "desk",
    "detail",
    "development",
    "digestion",
    "dime",
    "dinner",
    "direction",
    "dirt",
    "discovery",
    "discussion",
    "disease",
    "disgust",


"discussion",
    "disease",
    "disgust",
    "distance",
    "distribution",
    "division",
    "doctor",
    "dog",
    "door",
    "drain",
    "drawer",
    "dress",
    "drink",
    "driving",
    "dust",
    "ear",
    "earth",
    "edge",
    "education",
    "effect",
    "egg",
    "end",
    "energy",
    "engine",
    "error",
    "event",
    "example",
    "exchange",
    "existence",
    "expansion",
    "experience",
    "expert",
    "eye",
    "face",
    "fact",
    "fall",
    "family",
    "farm",
    "father",
    "fear",
    "feeling",
    "field",
    "finger",
    "fire",
    "fish",
    "flag",
    "flight",
    "floor",
    "flower",
    "fold",
    "food",
    "football",
    "force",
    "form",
    "frame",
    "friend",
    "frog",
    "fruit",
    "fuel",
    "furniture",
    "game",
    "garden",
    "gate",
    "girl",
    "glass",
    "glove",
    "goat",
    "gold",
    "government",
    "grade",
    "grain",
    "grass",
    "green",
    "grip",
    "group",
    "growth",
    "guide",


"group",
    "growth",
    "guide",
    "guitar",
    "hair",
    "hall",
    "hand",
    "harbor",
    "harmony",
    "hat",
    "head",
    "health",
    "heart",
    "heat",
    "hill",
    "history",
    "hobbies",
    "hole",
    "hope",
    "horn",
    "horse",
    "hospital",
    "hour",
    "house",
    "humor",
    "idea",
    "impulse",
    "income",
    "increase",
    "industry",
    "ink",
    "insect",
    "instrument",
    "insurance",
    "interest",
    "invention",
    "iron",
    "island",
    "jelly",
    "jet",
    "jewel",
    "join",
    "judge",
    "juice",
    "jump",
    "kettle",
    "key",
    "kick",
    "kiss",
    "kitten",
    "knee",
    "knife",
    "knowledge",
    "land",
    "language",
    "laugh",
    "law",
    "lead",
    "learning",
    "leather",
    "leg",
    "lettuce",
    "level",
    "library",
    "lift",
    "light",
    "limit",
    "line",
    "linen",
    "lip",
    "liquid",
    "list",
    "look",
    "loss",
    "love",


"list",
    "look",
    "loss",
    "love",
    "lunch",
    "machine",
    "man",
    "manager",
    "map",
    "marble",
    "mark",
    "market",
    "mass",
    "match",
    "meal",
    "measure",
    "meat",
    "meeting",
    "memory",
    "metal",
    "middle",
    "milk",
    "mind",
    "mine",
    "minute",
    "mist",
    "mitten",
    "mom",
    "money",
    "monkey",
    "month",
    "moon",
    "morning",
    "mother",
    "motion",
    "mountain",
    "mouth",
    "muscle",
    "music",
    "nail",
    "name",
    "nation",
    "neck",
    "need",
    "news",
    "night",
    "noise",
    "note",
    "number",
    "nut",
    "observation",
    "offer",
    "oil",
    "operation",
    "opinion",
    "orange",
    "order",
    "organization",
    "ornament",
    "oven",
    "page",
    "pail",
    "pain",
    "paint",
    "pan",
    "pancake",
    "paper",
    "parcel",
    "parent",
    "part",
    "passenger",
    "paste",
    "payment",
    "peace",
    "pear",
    "pen",
    "pencil",


"pear",
    "pen",
    "pencil",
    "person",
    "pest",
    "pet",
    "picture",
    "pie",
    "pin",
    "pipe",
    "pizza",
    "place",
    "plane",
    "plant",
    "plastic",
    "plate",
    "play",
    "pleasure",
    "plot",
    "plough",
    "pocket",
    "point",
    "poison",
    "police",
    "pollution",
    "popcorn",
    "porter",
    "position",
    "pot",
    "potato",
    "powder",
    "power",
    "price",
    "print",
    "process",
    "produce",
    "product",
    "profit",
    "property",
    "prose",
    "protest",
    "pull",
    "pump",
    "punishment",
    "purpose",
    "push",
    "quarter",
    "question",
    "quiet",
    "quill",
    "quilt",
    "quince",
    "rabbit",
    "rail",
    "rain",
    "range",
    "rat",
    "rate",
    "ray",
    "reaction",
    "reading",
    "reason",
    "record",
    "regret",
    "relation",
    "religion",
    "representative",
    "request",
    "respect",
    "rest",
    "reward",
    "rhythm",
    "rice",
    "river",
    "road",


"rice",
    "river",
    "road",
    "roll",
    "room",
    "root",
    "rose",
    "route",
    "rub",
    "rule",
    "run",
    "sack",
    "sail",
    "salt",
    "sand",
    "scale",
    "scarecrow",
    "scarf",
    "scene",
    "scent",
    "school",
    "science",
    "scissors",
    "screw",
    "sea",
    "seat",
    "secretary",
    "seed",
    "selection",
    "self",
    "sense",
    "servant",
    "shade",
    "shake",
    "shame",
    "shape",
    "sheep",
    "sheet",
    "shelf",
    "ship",
    "shirt",
    "shock",
    "shoe",
    "shop",
    "show",
    "side",
    "sign",
    "silk",
    "sink",
    "sister",
    "size",
    "sky",
    "slave",
    "sleep",
    "smash",
    "smell",
    "smile",
    "smoke",
    "snail",
    "snake",
    "sneeze",
    "snow",
    "soap",
    "society",
    "sock",
    "soda",
    "sofa",
    "son",
    "song",
    "sort",
    "sound",
    "soup",
    "space",
    "spark",


"sound",
    "soup",
    "space",
    "spark",
    "speed",
    "sponge",
    "spoon",
    "spray",
    "spring",
    "spy",
    "square",
    "stamp",
    "star",
    "start",
    "statement",
    "station",
    "steam",
    "steel",
    "stem",
    "step",
    "stew",
    "stick",
    "stitch",
    "stocking",
    "stomach",
    "stone",
    "stop",
    "store",
    "story",
    "stove",
    "stranger",
    "straw",
    "stream",
    "street",
    "stretch",
    "string",
    "structure",
    "substance",
    "sugar",
    "suggestion",
    "suit",
    "summer",
    "sun",
    "support",
    "surprise",
    "sweater",
    "swim",
    "system",
    "table",
    "tail",
    "talk",
    "tank",
    "taste",
    "tax",
    "tea",
    "teaching",
    "team",
    "tendency",
    "test",
    "texture",
    "theory",
    "thing",
    "thought",
    "thread",
    "throat",
    "thumb",
    "thunder",
    "ticket",
    "time",
    "tin",
    "title",
    "toad",
    "toe",
    "tooth",


"toad",
    "toe",
    "tooth",
    "toothpaste",
    "touch",
    "town",
    "toy",
    "trade",
    "train",
    "transport",
    "tray",
    "treatment",
    "tree",
    "trick",
    "trip",
    "trouble",
    "trousers",
    "truck",
    "tub",
    "turkey",
    "turn",
    "twist",
    "umbrella",
    "uncle",
    "underwear",
    "unit",
    "use",
    "vacation",
    "value",
    "van",
    "vase",
    "vegetable",
    "veil",
    "vein",
    "verse",
    "vessel",
    "view",
    "visitor",
    "voice",
    "volcano",
    "walk",
    "wall",
    "war",
    "wash",
    "waste",
    "watch",
    "water",
    "wave",
    "wax",
    "way",
    "wealth",
    "weather",
    "week",
    "weight",
    "wheel",
    "whip",
    "whistle",
    "window",
    "wine",
    "wing",
    "winter",
    "wire",
    "wish",
    "woman",
    "wood",
    "wool",
    "word",
    "work",
    "worm",
    "wound",
    "wrist",
    "writer",
    "yard",


"wrist",
    "writer",
    "yard",
    "yoke",
    "zebra",
    "zinc",
    "zipper",
    "zone",
]
[docs]def random_name(prefix: str = "test") -> str:
    """Generate a random name."""
    adjective = random.choice(adjectives)
    noun = random.choice(nouns)
    number = random.randint(1, 100)
    return f"{prefix}-{adjective}-{noun}-{number}"


Source code for langchain.smith.evaluation.config
"""Configuration for run evaluators."""
from typing import Any, Dict, List, Optional, Union
from langsmith import RunEvaluator
from langchain.evaluation.criteria.eval_chain import CRITERIA_TYPE
from langchain.evaluation.embedding_distance.base import (
    EmbeddingDistance as EmbeddingDistanceEnum,
)
from langchain.evaluation.schema import EvaluatorType, StringEvaluator
from langchain.evaluation.string_distance.base import (
    StringDistance as StringDistanceEnum,
)
from langchain.pydantic_v1 import BaseModel, Field
from langchain.schema.embeddings import Embeddings
from langchain.schema.language_model import BaseLanguageModel
from langchain.schema.prompt_template import BasePromptTemplate
[docs]class EvalConfig(BaseModel):
    """Configuration for a given run evaluator.
    Parameters
    ----------
    evaluator_type : EvaluatorType
        The type of evaluator to use.
    Methods
    -------
    get_kwargs()
        Get the keyword arguments for the evaluator configuration.
    """
    evaluator_type: EvaluatorType
[docs]    def get_kwargs(self) -> Dict[str, Any]:
        """Get the keyword arguments for the load_evaluator call.
        Returns
        -------
        Dict[str, Any]
            The keyword arguments for the load_evaluator call.
        """
        kwargs = {}
        for field, val in self:
            if field == "evaluator_type":
                continue
            elif val is None:
                continue
            kwargs[field] = val
        return kwargs
[docs]class RunEvalConfig(BaseModel):
    """Configuration for a run evaluation.
    Parameters
    ----------
    evaluators : List[Union[EvaluatorType, EvalConfig]]
        Configurations for which evaluators to apply to the dataset run.


Configurations for which evaluators to apply to the dataset run.
        Each can be the string of an :class:`EvaluatorType <langchain.evaluation.schema.EvaluatorType>`, such
        as EvaluatorType.QA, the evaluator type string ("qa"), or a configuration for a
        given evaluator (e.g., :class:`RunEvalConfig.QA <langchain.smith.evaluation.config.RunEvalConfig.QA>`).
    custom_evaluators : Optional[List[Union[RunEvaluator, StringEvaluator]]]
        Custom evaluators to apply to the dataset run.
    reference_key : Optional[str]
        The key in the dataset run to use as the reference string.
        If not provided, it will be inferred automatically.
    prediction_key : Optional[str]
        The key from the traced run's outputs dictionary to use to
        represent the prediction. If not provided, it will be inferred
        automatically.
    input_key : Optional[str]
        The key from the traced run's inputs dictionary to use to represent the
        input. If not provided, it will be inferred automatically.
    eval_llm : Optional[BaseLanguageModel]
        The language model to pass to any evaluators that use a language model.
    """  # noqa: E501
    evaluators: List[Union[EvaluatorType, str, EvalConfig]] = Field(
        default_factory=list
    )
    """Configurations for which evaluators to apply to the dataset run.
    Each can be the string of an
    :class:`EvaluatorType <langchain.evaluation.schema.EvaluatorType>`, such
    as `EvaluatorType.QA`, the evaluator type string ("qa"), or a configuration for a
    given evaluator
    (e.g.,


given evaluator
    (e.g.,
    :class:`RunEvalConfig.QA <langchain.smith.evaluation.config.RunEvalConfig.QA>`)."""  # noqa: E501
    custom_evaluators: Optional[List[Union[RunEvaluator, StringEvaluator]]] = None
    """Custom evaluators to apply to the dataset run."""
    reference_key: Optional[str] = None
    """The key in the dataset run to use as the reference string.
    If not provided, we will attempt to infer automatically."""
    prediction_key: Optional[str] = None
    """The key from the traced run's outputs dictionary to use to
    represent the prediction. If not provided, it will be inferred
    automatically."""
    input_key: Optional[str] = None
    """The key from the traced run's inputs dictionary to use to represent the
    input. If not provided, it will be inferred automatically."""
    eval_llm: Optional[BaseLanguageModel] = None
    """The language model to pass to any evaluators that require one."""
    class Config:
        arbitrary_types_allowed = True
[docs]    class Criteria(EvalConfig):
        """Configuration for a reference-free criteria evaluator.
        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        criteria: Optional[CRITERIA_TYPE] = None
        llm: Optional[BaseLanguageModel] = None
        evaluator_type: EvaluatorType = EvaluatorType.CRITERIA
        def __init__(
            self, criteria: Optional[CRITERIA_TYPE] = None, **kwargs: Any
        ) -> None:
            super().__init__(criteria=criteria, **kwargs)


) -> None:
            super().__init__(criteria=criteria, **kwargs)
[docs]    class LabeledCriteria(EvalConfig):
        """Configuration for a labeled (with references) criteria evaluator.
        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        criteria: Optional[CRITERIA_TYPE] = None
        llm: Optional[BaseLanguageModel] = None
        evaluator_type: EvaluatorType = EvaluatorType.LABELED_CRITERIA
        def __init__(
            self, criteria: Optional[CRITERIA_TYPE] = None, **kwargs: Any
        ) -> None:
            super().__init__(criteria=criteria, **kwargs)
[docs]    class EmbeddingDistance(EvalConfig):
        """Configuration for an embedding distance evaluator.
        Parameters
        ----------
        embeddings : Optional[Embeddings]
            The embeddings to use for computing the distance.
        distance_metric : Optional[EmbeddingDistanceEnum]
            The distance metric to use for computing the distance.
        """
        evaluator_type: EvaluatorType = EvaluatorType.EMBEDDING_DISTANCE
        embeddings: Optional[Embeddings] = None
        distance_metric: Optional[EmbeddingDistanceEnum] = None
        class Config:
            arbitrary_types_allowed = True
[docs]    class StringDistance(EvalConfig):
        """Configuration for a string distance evaluator.
        Parameters
        ----------
        distance : Optional[StringDistanceEnum]
            The string distance metric to use.
        """
        evaluator_type: EvaluatorType = EvaluatorType.STRING_DISTANCE
        distance: Optional[StringDistanceEnum] = None


distance: Optional[StringDistanceEnum] = None
        """The string distance metric to use.
            damerau_levenshtein: The Damerau-Levenshtein distance.
            levenshtein: The Levenshtein distance.
            jaro: The Jaro distance.
            jaro_winkler: The Jaro-Winkler distance.
        """
        normalize_score: bool = True
        """Whether to normalize the distance to between 0 and 1.
        Applies only to the Levenshtein and Damerau-Levenshtein distances."""
[docs]    class QA(EvalConfig):
        """Configuration for a QA evaluator.
        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        evaluator_type: EvaluatorType = EvaluatorType.QA
        llm: Optional[BaseLanguageModel] = None
        prompt: Optional[BasePromptTemplate] = None
[docs]    class ContextQA(EvalConfig):
        """Configuration for a context-based QA evaluator.
        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        evaluator_type: EvaluatorType = EvaluatorType.CONTEXT_QA
        llm: Optional[BaseLanguageModel] = None
        prompt: Optional[BasePromptTemplate] = None
[docs]    class CoTQA(EvalConfig):
        """Configuration for a context-based QA evaluator.
        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]


Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        evaluator_type: EvaluatorType = EvaluatorType.CONTEXT_QA
        llm: Optional[BaseLanguageModel] = None
        prompt: Optional[BasePromptTemplate] = None
[docs]    class JsonValidity(EvalConfig):
        """Configuration for a json validity evaluator.
        Parameters
        ----------
        """
        evaluator_type: EvaluatorType = EvaluatorType.JSON_VALIDITY
[docs]    class JsonEqualityEvaluator(EvalConfig):
        """Configuration for a json equality evaluator.
        Parameters
        ----------
        """
        evaluator_type: EvaluatorType = EvaluatorType.JSON_EQUALITY
[docs]    class ExactMatch(EvalConfig):
        """Configuration for an exact match string evaluator.
        Parameters
        ----------
        ignore_case : bool
            Whether to ignore case when comparing strings.
        ignore_punctuation : bool
            Whether to ignore punctuation when comparing strings.
        ignore_numbers : bool
            Whether to ignore numbers when comparing strings.
        """
        evaluator_type: EvaluatorType = EvaluatorType.STRING_DISTANCE
        ignore_case: bool = False
        ignore_punctuation: bool = False
        ignore_numbers: bool = False
[docs]    class RegexMatch(EvalConfig):
        """Configuration for a regex match string evaluator.
        Parameters
        ----------
        flags : int
            The flags to pass to the regex. Example: re.IGNORECASE.
        """
        evaluator_type: EvaluatorType = EvaluatorType.REGEX_MATCH
        flags: int = 0
    # TODO: Trajectory


Source code for langchain.smith.evaluation.progress
"""A simple progress bar for the console."""
import threading
from typing import Any, Dict, Optional, Sequence
from uuid import UUID
from langchain.callbacks import base as base_callbacks
from langchain.schema.document import Document
from langchain.schema.output import LLMResult
[docs]class ProgressBarCallback(base_callbacks.BaseCallbackHandler):
    """A simple progress bar for the console."""
[docs]    def __init__(self, total: int, ncols: int = 50, **kwargs: Any):
        """Initialize the progress bar.
        Args:
            total: int, the total number of items to be processed.
            ncols: int, the character width of the progress bar.
        """
        self.total = total
        self.ncols = ncols
        self.counter = 0
        self.lock = threading.Lock()
        self._print_bar()
[docs]    def increment(self) -> None:
        """Increment the counter and update the progress bar."""
        with self.lock:
            self.counter += 1
            self._print_bar()
    def _print_bar(self) -> None:
        """Print the progress bar to the console."""
        progress = self.counter / self.total
        arrow = "-" * int(round(progress * self.ncols) - 1) + ">"
        spaces = " " * (self.ncols - len(arrow))
        print(f"\r[{arrow + spaces}] {self.counter}/{self.total}", end="")
[docs]    def on_chain_end(
        self,
        outputs: Dict[str, Any],
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:


) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_retriever_end(
        self,
        documents: Sequence[Document],
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_llm_end(
        self,
        response: LLMResult,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_tool_end(
        self,
        output: str,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()


Source code for langchain.smith.evaluation.runner_utils
"""Utilities for running language models or Chains over datasets."""
from __future__ import annotations
import functools
import inspect
import logging
import warnings
from enum import Enum
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    List,
    Optional,
    Sequence,
    Tuple,
    Union,
    cast,
)
from langsmith import Client, RunEvaluator
from langsmith.schemas import Dataset, DataType, Example
from langchain.callbacks.manager import Callbacks
from langchain.callbacks.tracers.evaluation import (
    EvaluatorCallbackHandler,
    wait_for_all_evaluators,
)
from langchain.callbacks.tracers.langchain import LangChainTracer
from langchain.chains.base import Chain
from langchain.evaluation.loading import load_evaluator
from langchain.evaluation.schema import (
    EvaluatorType,
    PairwiseStringEvaluator,
    StringEvaluator,
)
from langchain.schema import ChatResult, LLMResult
from langchain.schema.language_model import BaseLanguageModel
from langchain.schema.messages import BaseMessage, messages_from_dict
from langchain.schema.runnable import Runnable, RunnableConfig, RunnableLambda
from langchain.schema.runnable import config as runnable_config
from langchain.schema.runnable import utils as runnable_utils
from langchain.smith import evaluation as smith_eval
from langchain.smith.evaluation import config as smith_eval_config
from langchain.smith.evaluation import name_generation, progress
if TYPE_CHECKING:
    import pandas as pd
logger = logging.getLogger(__name__)
MODEL_OR_CHAIN_FACTORY = Union[
    Callable[[], Union[Chain, Runnable]],
    BaseLanguageModel,
    Callable[[dict], Any],
    Runnable,
    Chain,
]


Callable[[dict], Any],
    Runnable,
    Chain,
]
MCF = Union[Callable[[], Union[Chain, Runnable]], BaseLanguageModel]
[docs]class InputFormatError(Exception):
    """Raised when the input format is invalid."""
## Shared Utilities
[docs]class TestResult(dict):
    """A dictionary of the results of a single test run."""
[docs]    def get_aggregate_feedback(
        self, quantiles: Optional[Sequence[float]] = None
    ) -> pd.DataFrame:
        """Return quantiles for the feedback scores.
        This method calculates and prints the quantiles for the feedback scores
        across all feedback keys.
        Returns:
            A DataFrame containing the quantiles for each feedback key.
        """
        df = self.to_dataframe()
        feedback_cols = [
            col for col in df.columns if col not in ["input", "output", "reference"]
        ]
        _quantiles = df[feedback_cols].quantile(
            quantiles or [0.25, 0.5, 0.75], numeric_only=True
        )
        _quantiles.loc["mean"] = df[feedback_cols].mean()
        _quantiles.loc["mode"] = df[feedback_cols].mode().iloc[0]
        return _quantiles.transpose()
[docs]    def to_dataframe(self) -> pd.DataFrame:
        """Convert the results to a dataframe."""
        try:
            import pandas as pd
        except ImportError as e:
            raise ImportError(
                "Pandas is required to convert the results to a dataframe."
                " to install pandas, run `pip install pandas`."
            ) from e
        indices = []
        records = []
        for example_id, result in self["results"].items():


records = []
        for example_id, result in self["results"].items():
            feedback = result["feedback"]
            r = {
                **{f.key: f.score for f in feedback},
                "input": result["input"],
                "output": result["output"],
            }
            if "reference" in result:
                r["reference"] = result["reference"]
            records.append(r)
            indices.append(example_id)
        return pd.DataFrame(records, index=indices)
def _wrap_in_chain_factory(
    llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,
    dataset_name: str = "<my_dataset>",
) -> MCF:
    """Forgive the user if they pass in a chain without memory instead of a chain
    factory. It's a common mistake. Raise a more helpful error message as well."""
    if isinstance(llm_or_chain_factory, Chain):
        chain = llm_or_chain_factory
        chain_class = chain.__class__.__name__
        if llm_or_chain_factory.memory is not None:
            memory_class = chain.memory.__class__.__name__
            raise ValueError(
                "Cannot directly evaluate a chain with stateful memory."
                " To evaluate this chain, pass in a chain constructor"
                " that initializes fresh memory each time it is called."
                "  This will safegaurd against information"
                " leakage between dataset examples."
                "\nFor example:\n\n"
                "def chain_constructor():\n"
                f"    new_memory = {memory_class}(...)\n"
                f"    return {chain_class}"
                "(memory=new_memory, ...)\n\n"
                f'run_on_dataset("{dataset_name}", chain_constructor, ...)'
            )
        return lambda: chain


)
        return lambda: chain
    elif isinstance(llm_or_chain_factory, BaseLanguageModel):
        return llm_or_chain_factory
    elif isinstance(llm_or_chain_factory, Runnable):
        # Memory may exist here, but it's not elegant to check all those cases.
        lcf = llm_or_chain_factory
        return lambda: lcf
    elif callable(llm_or_chain_factory):
        try:
            _model = llm_or_chain_factory()  # type: ignore[call-arg]
        except TypeError:
            # It's an arbitrary function, wrap it in a RunnableLambda
            user_func = cast(Callable, llm_or_chain_factory)
            sig = inspect.signature(user_func)
            logger.info(f"Wrapping function {sig} as RunnableLambda.")
            wrapped = RunnableLambda(user_func)
            return lambda: wrapped
        constructor = cast(Callable, llm_or_chain_factory)
        if isinstance(_model, BaseLanguageModel):
            # It's not uncommon to do an LLM constructor instead of raw LLM,
            # so we'll unpack it for the user.
            return _model
        elif not isinstance(_model, Runnable):
            # This is unlikely to happen - a constructor for a model function
            return lambda: RunnableLambda(constructor)
        else:
            # Typical correct case
            return constructor  # noqa
    return llm_or_chain_factory
def _get_prompt(inputs: Dict[str, Any]) -> str:
    """Get prompt from inputs.
    Args:
        inputs: The input dictionary.
    Returns:
        A string prompt.
    Raises:
        InputFormatError: If the input format is invalid.
    """
    if not inputs:
        raise InputFormatError("Inputs should not be empty.")
    prompts = []


raise InputFormatError("Inputs should not be empty.")
    prompts = []
    if "prompt" in inputs:
        if not isinstance(inputs["prompt"], str):
            raise InputFormatError(
                "Expected string for 'prompt', got"
                f" {type(inputs['prompt']).__name__}"
            )
        prompts = [inputs["prompt"]]
    elif "prompts" in inputs:
        if not isinstance(inputs["prompts"], list) or not all(
            isinstance(i, str) for i in inputs["prompts"]
        ):
            raise InputFormatError(
                "Expected list of strings for 'prompts',"
                f" got {type(inputs['prompts']).__name__}"
            )
        prompts = inputs["prompts"]
    elif len(inputs) == 1:
        prompt_ = next(iter(inputs.values()))
        if isinstance(prompt_, str):
            prompts = [prompt_]
        elif isinstance(prompt_, list) and all(isinstance(i, str) for i in prompt_):
            prompts = prompt_
        else:
            raise InputFormatError(f"LLM Run expects string prompt input. Got {inputs}")
    else:
        raise InputFormatError(
            f"LLM Run expects 'prompt' or 'prompts' in inputs. Got {inputs}"
        )
    if len(prompts) == 1:
        return prompts[0]
    else:
        raise InputFormatError(
            f"LLM Run expects single prompt input. Got {len(prompts)} prompts."
        )
def _get_messages(inputs: Dict[str, Any]) -> List[BaseMessage]:
    """Get Chat Messages from inputs.
    Args:
        inputs: The input dictionary.
    Returns:
        A list of chat messages.


inputs: The input dictionary.
    Returns:
        A list of chat messages.
    Raises:
        InputFormatError: If the input format is invalid.
    """
    if not inputs:
        raise InputFormatError("Inputs should not be empty.")
    if "messages" in inputs:
        single_input = inputs["messages"]
    elif len(inputs) == 1:
        single_input = next(iter(inputs.values()))
    else:
        raise InputFormatError(
            f"Chat Run expects 'messages' in inputs when example has multiple"
            f" input keys. Got {inputs}"
        )
    if isinstance(single_input, list) and all(
        isinstance(i, dict) for i in single_input
    ):
        raw_messages = [single_input]
    elif isinstance(single_input, list) and all(
        isinstance(i, list) for i in single_input
    ):
        raw_messages = single_input
    else:
        raise InputFormatError(
            f"Chat Run expects List[dict] or List[List[dict]] values for"
            f" 'messages' key input. Got {inputs}"
        )
    if len(raw_messages) == 1:
        return messages_from_dict(raw_messages[0])
    else:
        raise InputFormatError(
            f"Chat Run expects single List[dict] or List[List[dict]] 'messages'"
            f" input. Got {len(raw_messages)} messages from inputs {inputs}"
        )
## Shared data validation utilities
def _validate_example_inputs_for_language_model(
    first_example: Example,
    input_mapper: Optional[Callable[[Dict], Any]],
) -> None:
    if input_mapper:
        prompt_input = input_mapper(first_example.inputs)


if input_mapper:
        prompt_input = input_mapper(first_example.inputs)
        if not isinstance(prompt_input, str) and not (
            isinstance(prompt_input, list)
            and all(isinstance(msg, BaseMessage) for msg in prompt_input)
        ):
            raise InputFormatError(
                "When using an input_mapper to prepare dataset example inputs"
                " for an LLM or chat model, the output must a single string or"
                " a list of chat messages."
                f"\nGot: {prompt_input} of type {type(prompt_input)}."
            )
    else:
        try:
            _get_prompt(first_example.inputs)
        except InputFormatError:
            try:
                _get_messages(first_example.inputs)
            except InputFormatError:
                raise InputFormatError(
                    "Example inputs do not match language model input format. "
                    "Expected a dictionary with messages or a single prompt."
                    f" Got: {first_example.inputs}"
                    " Please update your dataset OR provide an input_mapper"
                    " to convert the example.inputs to a compatible format"
                    " for the llm or chat model you wish to evaluate."
                )
def _validate_example_inputs_for_chain(
    first_example: Example,
    chain: Chain,
    input_mapper: Optional[Callable[[Dict], Any]],
) -> None:
    """Validate that the example inputs match the chain input keys."""
    if input_mapper:
        first_inputs = input_mapper(first_example.inputs)
        missing_keys = set(chain.input_keys).difference(first_inputs)
        if not isinstance(first_inputs, dict):
            raise InputFormatError(
                "When using an input_mapper to prepare dataset example"
                " inputs for a chain, the mapped value must be a dictionary."


" inputs for a chain, the mapped value must be a dictionary."
                f"\nGot: {first_inputs} of type {type(first_inputs)}."
            )
        if missing_keys:
            raise InputFormatError(
                "Missing keys after loading example using input_mapper."
                f"\nExpected: {chain.input_keys}. Got: {first_inputs.keys()}"
            )
    else:
        first_inputs = first_example.inputs
        missing_keys = set(chain.input_keys).difference(first_inputs)
        if len(first_inputs) == 1 and len(chain.input_keys) == 1:
            # We can pass this through the run method.
            # Refrain from calling to validate.
            pass
        elif missing_keys:
            raise InputFormatError(
                "Example inputs missing expected chain input keys."
                " Please provide an input_mapper to convert the example.inputs"
                " to a compatible format for the chain you wish to evaluate."
                f"Expected: {chain.input_keys}. "
                f"Got: {first_inputs.keys()}"
            )
def _validate_example_inputs(
    example: Example,
    llm_or_chain_factory: MCF,
    input_mapper: Optional[Callable[[Dict], Any]],
) -> None:
    """Validate that the example inputs are valid for the model."""
    if isinstance(llm_or_chain_factory, BaseLanguageModel):
        _validate_example_inputs_for_language_model(example, input_mapper)
    else:
        chain = llm_or_chain_factory()
        if isinstance(chain, Chain):
            # Otherwise it's a runnable
            _validate_example_inputs_for_chain(example, chain, input_mapper)
        elif isinstance(chain, Runnable):
            logger.debug(f"Skipping input validation for {chain}")
## Shared Evaluator Setup Utilities
def _setup_evaluation(


## Shared Evaluator Setup Utilities
def _setup_evaluation(
    llm_or_chain_factory: MCF,
    examples: List[Example],
    evaluation: Optional[smith_eval.RunEvalConfig],
    data_type: DataType,
) -> Optional[List[RunEvaluator]]:
    """Configure the evaluators to run on the results of the chain."""
    if evaluation:
        if isinstance(llm_or_chain_factory, BaseLanguageModel):
            run_inputs, run_outputs = None, None
            run_type = "llm"
        else:
            run_type = "chain"
            if data_type in (DataType.chat, DataType.llm):
                val = data_type.value if isinstance(data_type, Enum) else data_type
                raise ValueError(
                    "Cannot evaluate a chain on dataset with "
                    f"data_type={val}. "
                    "Please specify a dataset with the default 'kv' data type."
                )
            chain = llm_or_chain_factory()
            run_inputs = chain.input_keys if isinstance(chain, Chain) else None
            run_outputs = chain.output_keys if isinstance(chain, Chain) else None
        run_evaluators = _load_run_evaluators(
            evaluation,
            run_type,
            data_type,
            list(examples[0].outputs) if examples[0].outputs else None,
            run_inputs,
            run_outputs,
        )
    else:
        # TODO: Create a default helpfulness evaluator
        run_evaluators = None
    return run_evaluators
def _determine_input_key(
    config: smith_eval.RunEvalConfig,
    run_inputs: Optional[List[str]],
) -> Optional[str]:
    input_key = None
    if config.input_key:
        input_key = config.input_key


if config.input_key:
        input_key = config.input_key
        if run_inputs and input_key not in run_inputs:
            raise ValueError(f"Input key {input_key} not in run inputs {run_inputs}")
    elif run_inputs and len(run_inputs) == 1:
        input_key = run_inputs[0]
    elif run_inputs is not None and len(run_inputs) > 1:
        raise ValueError(
            f"Must specify input key for model with multiple inputs: {run_inputs}"
        )
    return input_key
def _determine_prediction_key(
    config: smith_eval.RunEvalConfig,
    run_outputs: Optional[List[str]],
) -> Optional[str]:
    prediction_key = None
    if config.prediction_key:
        prediction_key = config.prediction_key
        if run_outputs and prediction_key not in run_outputs:
            raise ValueError(
                f"Prediction key {prediction_key} not in run outputs {run_outputs}"
            )
    elif run_outputs and len(run_outputs) == 1:
        prediction_key = run_outputs[0]
    elif run_outputs is not None and len(run_outputs) > 1:
        raise ValueError(
            f"Must specify prediction key for model"
            f" with multiple outputs: {run_outputs}"
        )
    return prediction_key
def _determine_reference_key(
    config: smith_eval.RunEvalConfig,
    example_outputs: Optional[List[str]],
) -> Optional[str]:
    if config.reference_key:
        reference_key = config.reference_key
        if example_outputs and reference_key not in example_outputs:
            raise ValueError(
                f"Reference key {reference_key} not in Dataset"
                f" example outputs: {example_outputs}"
            )
    elif example_outputs and len(example_outputs) == 1:


)
    elif example_outputs and len(example_outputs) == 1:
        reference_key = list(example_outputs)[0]
    else:
        reference_key = None
    return reference_key
def _construct_run_evaluator(
    eval_config: Union[EvaluatorType, str, smith_eval_config.EvalConfig],
    eval_llm: Optional[BaseLanguageModel],
    run_type: str,
    data_type: DataType,
    example_outputs: Optional[List[str]],
    reference_key: Optional[str],
    input_key: Optional[str],
    prediction_key: Optional[str],
) -> RunEvaluator:
    if isinstance(eval_config, (EvaluatorType, str)):
        if not isinstance(eval_config, EvaluatorType):
            eval_config = EvaluatorType(eval_config)
        evaluator_ = load_evaluator(eval_config, llm=eval_llm)
        eval_type_tag = eval_config.value
    else:
        kwargs = {"llm": eval_llm, **eval_config.get_kwargs()}
        evaluator_ = load_evaluator(eval_config.evaluator_type, **kwargs)
        eval_type_tag = eval_config.evaluator_type.value
    if isinstance(evaluator_, StringEvaluator):
        if evaluator_.requires_reference and reference_key is None:
            raise ValueError(
                f"Must specify reference_key in smith_eval.RunEvalConfig to use"
                f" evaluator of type {eval_type_tag} with"
                f" dataset with multiple output keys: {example_outputs}."
            )
        run_evaluator = smith_eval.StringRunEvaluatorChain.from_run_and_data_type(
            evaluator_,
            run_type,
            data_type,
            input_key=input_key,
            prediction_key=prediction_key,
            reference_key=reference_key,
            tags=[eval_type_tag],
        )


reference_key=reference_key,
            tags=[eval_type_tag],
        )
    elif isinstance(evaluator_, PairwiseStringEvaluator):
        raise NotImplementedError(
            f"Run evaluator for {eval_type_tag} is not implemented."
            " PairwiseStringEvaluators compare the outputs of two different models"
            " rather than the output of a single model."
            " Did you mean to use a StringEvaluator instead?"
            "\nSee: https://python.langchain.com/docs/guides/evaluation/string/"
        )
    else:
        raise NotImplementedError(
            f"Run evaluator for {eval_type_tag} is not implemented"
        )
    return run_evaluator
def _get_keys(
    config: smith_eval.RunEvalConfig,
    run_inputs: Optional[List[str]],
    run_outputs: Optional[List[str]],
    example_outputs: Optional[List[str]],
) -> Tuple[Optional[str], Optional[str], Optional[str]]:
    input_key = _determine_input_key(config, run_inputs)
    prediction_key = _determine_prediction_key(config, run_outputs)
    reference_key = _determine_reference_key(config, example_outputs)
    return input_key, prediction_key, reference_key
def _load_run_evaluators(
    config: smith_eval.RunEvalConfig,
    run_type: str,
    data_type: DataType,
    example_outputs: Optional[List[str]],
    run_inputs: Optional[List[str]],
    run_outputs: Optional[List[str]],
) -> List[RunEvaluator]:
    """
    Load run evaluators from a configuration.
    Args:
        config: Configuration for the run evaluators.
    Returns:
        A list of run evaluators.
    """
    run_evaluators = []


A list of run evaluators.
    """
    run_evaluators = []
    input_key, prediction_key, reference_key = None, None, None
    if (
        config.evaluators
        or any([isinstance(e, EvaluatorType) for e in config.evaluators])
        or (
            config.custom_evaluators
            and any([isinstance(e, StringEvaluator) for e in config.custom_evaluators])
        )
    ):
        input_key, prediction_key, reference_key = _get_keys(
            config, run_inputs, run_outputs, example_outputs
        )
    for eval_config in config.evaluators:
        run_evaluator = _construct_run_evaluator(
            eval_config,
            config.eval_llm,
            run_type,
            data_type,
            example_outputs,
            reference_key,
            input_key,
            prediction_key,
        )
        run_evaluators.append(run_evaluator)
    custom_evaluators = config.custom_evaluators or []
    for custom_evaluator in custom_evaluators:
        if isinstance(custom_evaluator, RunEvaluator):
            run_evaluators.append(custom_evaluator)
        elif isinstance(custom_evaluator, StringEvaluator):
            run_evaluators.append(
                smith_eval.StringRunEvaluatorChain.from_run_and_data_type(
                    custom_evaluator,
                    run_type,
                    data_type,
                    input_key=input_key,
                    prediction_key=prediction_key,
                    reference_key=reference_key,
                )
            )
        else:
            raise ValueError(
                f"Unsupported custom evaluator: {custom_evaluator}."
                f" Expected RunEvaluator or StringEvaluator."
            )
    return run_evaluators
### Async Helpers
async def _arun_llm(


return run_evaluators
### Async Helpers
async def _arun_llm(
    llm: BaseLanguageModel,
    inputs: Dict[str, Any],
    *,
    tags: Optional[List[str]] = None,
    callbacks: Callbacks = None,
    input_mapper: Optional[Callable[[Dict], Any]] = None,
) -> Union[str, BaseMessage]:
    """Asynchronously run the language model.
    Args:
        llm: The language model to run.
        inputs: The input dictionary.
        tags: Optional tags to add to the run.
        callbacks: Optional callbacks to use during the run.
        input_mapper: Optional function to map inputs to the expected format.
    Returns:
        The LLMResult or ChatResult.
    Raises:
        ValueError: If the LLM type is unsupported.
        InputFormatError: If the input format is invalid.
    """
    if input_mapper is not None:
        prompt_or_messages = input_mapper(inputs)
        if isinstance(prompt_or_messages, str):
            return await llm.apredict(
                prompt_or_messages, callbacks=callbacks, tags=tags
            )
        elif isinstance(prompt_or_messages, list) and all(
            isinstance(msg, BaseMessage) for msg in prompt_or_messages
        ):
            return await llm.apredict_messages(
                prompt_or_messages, callbacks=callbacks, tags=tags
            )
        else:
            raise InputFormatError(
                "Input mapper returned invalid format"
                f" {prompt_or_messages}"
                "\nExpected a single string or list of chat messages."
            )
    else:
        try:
            prompt = _get_prompt(inputs)
            llm_output: Union[str, BaseMessage] = await llm.apredict(


llm_output: Union[str, BaseMessage] = await llm.apredict(
                prompt, callbacks=callbacks, tags=tags
            )
        except InputFormatError:
            messages = _get_messages(inputs)
            llm_output = await llm.apredict_messages(
                messages, callbacks=callbacks, tags=tags
            )
    return llm_output
async def _arun_chain(
    chain: Union[Chain, Runnable],
    inputs: Dict[str, Any],
    callbacks: Callbacks,
    *,
    tags: Optional[List[str]] = None,
    input_mapper: Optional[Callable[[Dict], Any]] = None,
) -> Union[dict, str]:
    """Run a chain asynchronously on inputs."""
    inputs_ = inputs if input_mapper is None else input_mapper(inputs)
    if (
        isinstance(chain, Chain)
        and isinstance(inputs_, dict)
        and len(inputs_) == 1
        and chain.input_keys
    ):
        val = next(iter(inputs_.values()))
        output = await chain.acall(val, callbacks=callbacks, tags=tags)
    else:
        runnable_config = RunnableConfig(tags=tags or [], callbacks=callbacks)
        output = await chain.ainvoke(inputs_, config=runnable_config)
    return output
async def _arun_llm_or_chain(
    example: Example,
    config: RunnableConfig,
    *,
    llm_or_chain_factory: MCF,
    input_mapper: Optional[Callable[[Dict], Any]] = None,
) -> Union[dict, str, LLMResult, ChatResult]:
    """Asynchronously run the Chain or language model.
    Args:
        example: The example to run.
        llm_or_chain_factory: The Chain or language model constructor to run.


llm_or_chain_factory: The Chain or language model constructor to run.
        tags: Optional tags to add to the run.
        callbacks: Optional callbacks to use during the run.
        input_mapper: Optional function to map the input to the expected format.
    Returns:
        A list of outputs.
    """
    chain_or_llm = (
        "LLM" if isinstance(llm_or_chain_factory, BaseLanguageModel) else "Chain"
    )
    result = None
    try:
        if isinstance(llm_or_chain_factory, BaseLanguageModel):
            output: Any = await _arun_llm(
                llm_or_chain_factory,
                example.inputs,
                tags=config["tags"],
                callbacks=config["callbacks"],
                input_mapper=input_mapper,
            )
        else:
            chain = llm_or_chain_factory()
            output = await _arun_chain(
                chain,
                example.inputs,
                tags=config["tags"],
                callbacks=config["callbacks"],
                input_mapper=input_mapper,
            )
        result = output
    except Exception as e:
        logger.warning(
            f"{chain_or_llm} failed for example {example.id} "
            f"with inputs {example.inputs}"
            f"\n{repr(e)}"
        )
        result = {"Error": repr(e)}
    return result
## Sync Utilities
def _run_llm(
    llm: BaseLanguageModel,
    inputs: Dict[str, Any],
    callbacks: Callbacks,
    *,
    tags: Optional[List[str]] = None,
    input_mapper: Optional[Callable[[Dict], Any]] = None,
) -> Union[str, BaseMessage]:
    """
    Run the language model on the example.
    Args:


"""
    Run the language model on the example.
    Args:
        llm: The language model to run.
        inputs: The input dictionary.
        callbacks: The callbacks to use during the run.
        tags: Optional tags to add to the run.
        input_mapper: function to map to the inputs dictionary from an Example
    Returns:
        The LLMResult or ChatResult.
    Raises:
        ValueError: If the LLM type is unsupported.
        InputFormatError: If the input format is invalid.
    """
    if input_mapper is not None:
        prompt_or_messages = input_mapper(inputs)
        if isinstance(prompt_or_messages, str):
            llm_output: Union[str, BaseMessage] = llm.predict(
                prompt_or_messages, callbacks=callbacks, tags=tags
            )
        elif isinstance(prompt_or_messages, list) and all(
            isinstance(msg, BaseMessage) for msg in prompt_or_messages
        ):
            llm_output = llm.predict_messages(
                prompt_or_messages, callbacks=callbacks, tags=tags
            )
        else:
            raise InputFormatError(
                "Input mapper returned invalid format: "
                f" {prompt_or_messages}"
                "\nExpected a single string or list of chat messages."
            )
    else:
        try:
            llm_prompts = _get_prompt(inputs)
            llm_output = llm.predict(llm_prompts, callbacks=callbacks, tags=tags)
        except InputFormatError:
            llm_messages = _get_messages(inputs)
            llm_output = llm.predict_messages(llm_messages, callbacks=callbacks)
    return llm_output
def _run_chain(
    chain: Union[Chain, Runnable],
    inputs: Dict[str, Any],


chain: Union[Chain, Runnable],
    inputs: Dict[str, Any],
    callbacks: Callbacks,
    *,
    tags: Optional[List[str]] = None,
    input_mapper: Optional[Callable[[Dict], Any]] = None,
) -> Union[Dict, str]:
    """Run a chain on inputs."""
    inputs_ = inputs if input_mapper is None else input_mapper(inputs)
    if (
        isinstance(chain, Chain)
        and isinstance(inputs_, dict)
        and len(inputs_) == 1
        and chain.input_keys
    ):
        val = next(iter(inputs_.values()))
        output = chain(val, callbacks=callbacks, tags=tags)
    else:
        runnable_config = RunnableConfig(tags=tags or [], callbacks=callbacks)
        output = chain.invoke(inputs_, config=runnable_config)
    return output
def _run_llm_or_chain(
    example: Example,
    config: RunnableConfig,
    *,
    llm_or_chain_factory: MCF,
    input_mapper: Optional[Callable[[Dict], Any]] = None,
) -> Union[dict, str, LLMResult, ChatResult]:
    """
    Run the Chain or language model synchronously.
    Args:
        example: The example to run.
        llm_or_chain_factory: The Chain or language model constructor to run.
        tags: Optional tags to add to the run.
        callbacks: Optional callbacks to use during the run.
    Returns:
        Union[List[dict], List[str], List[LLMResult], List[ChatResult]]:
          The outputs of the model or chain.
    """
    chain_or_llm = (
        "LLM" if isinstance(llm_or_chain_factory, BaseLanguageModel) else "Chain"
    )


)
    result = None
    try:
        if isinstance(llm_or_chain_factory, BaseLanguageModel):
            output: Any = _run_llm(
                llm_or_chain_factory,
                example.inputs,
                config["callbacks"],
                tags=config["tags"],
                input_mapper=input_mapper,
            )
        else:
            chain = llm_or_chain_factory()
            output = _run_chain(
                chain,
                example.inputs,
                config["callbacks"],
                tags=config["tags"],
                input_mapper=input_mapper,
            )
        result = output
    except Exception as e:
        error_type = type(e).__name__
        logger.warning(
            f"{chain_or_llm} failed for example {example.id} "
            f"with inputs {example.inputs}"
            f"\nError Type: {error_type}, Message: {e}"
        )
        result = {"Error": repr(e)}
    return result
## Public API
def _prepare_eval_run(
    client: Client,
    dataset_name: str,
    llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,
    project_name: str,
    project_metadata: Optional[Dict[str, Any]] = None,
) -> Tuple[MCF, str, Dataset, List[Example]]:
    wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory, dataset_name)
    dataset = client.read_dataset(dataset_name=dataset_name)
    try:
        project = client.create_project(
            project_name,
            reference_dataset_id=dataset.id,
            project_extra={"metadata": project_metadata} if project_metadata else {},
        )
    except ValueError as e:
        if "already exists " not in str(e):
            raise e
        raise ValueError(