Source code for langchain.llms.openllm
from __future__ import annotations
import copy
import json
import logging
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    List,
    Literal,
    Optional,
    TypedDict,
    Union,
    overload,
)
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.pydantic_v1 import PrivateAttr
if TYPE_CHECKING:
    import openllm
ServerType = Literal["http", "grpc"]
[docs]class IdentifyingParams(TypedDict):
    """Parameters for identifying a model as a typed dict."""
    model_name: str
    model_id: Optional[str]
    server_url: Optional[str]
    server_type: Optional[ServerType]
    embedded: bool
    llm_kwargs: Dict[str, Any]
logger = logging.getLogger(__name__)
[docs]class OpenLLM(LLM):
    """OpenLLM, supporting both in-process model
    instance and remote OpenLLM servers.
    To use, you should have the openllm library installed:
    .. code-block:: bash
        pip install openllm
    Learn more at: https://github.com/bentoml/openllm
    Example running an LLM model locally managed by OpenLLM:
        .. code-block:: python
            from langchain.llms import OpenLLM
            llm = OpenLLM(
                model_name='flan-t5',
                model_id='google/flan-t5-large',
            )
            llm("What is the difference between a duck and a goose?")