Source code for langchain.llms.xinference
from typing import TYPE_CHECKING, Any, Dict, Generator, List, Mapping, Optional, Union
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
if TYPE_CHECKING:
    from xinference.client import RESTfulChatModelHandle, RESTfulGenerateModelHandle
    from xinference.model.llm.core import LlamaCppGenerateConfig
[docs]class Xinference(LLM):
    """Wrapper for accessing Xinference's large-scale model inference service.
    To use, you should have the xinference library installed:
    .. code-block:: bash
       pip install "xinference[all]"
    Check out: https://github.com/xorbitsai/inference
    To run, you need to start a Xinference supervisor on one server and Xinference workers on the other servers
    Example:
        To start a local instance of Xinference, run
        .. code-block:: bash
           $ xinference
        You can also deploy Xinference in a distributed cluster. Here are the steps:
        Starting the supervisor:
        .. code-block:: bash
           $ xinference-supervisor
        Starting the worker:
        .. code-block:: bash
           $ xinference-worker
    Then, launch a model using command line interface (CLI).
    Example:
    .. code-block:: bash
       $ xinference launch -n orca -s 3 -q q4_0
    It will return a model UID. Then, you can use Xinference with LangChain.
    Example:
    .. code-block:: python
        from langchain.llms import Xinference
        llm = Xinference(
            server_url="http://0.0.0.0:9997",