Source code for langchain.llms.deepsparse
# flake8: noqa
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
from langchain.pydantic_v1 import root_validator
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.llms.utils import enforce_stop_tokens
from langchain.schema.output import GenerationChunk
[docs]class DeepSparse(LLM):
    """Neural Magic DeepSparse LLM interface.
    To use, you should have the ``deepsparse`` or ``deepsparse-nightly``
    python package installed. See https://github.com/neuralmagic/deepsparse
    This interface let's you deploy optimized LLMs straight from the
    [SparseZoo](https://sparsezoo.neuralmagic.com/?useCase=text_generation)
    Example:
        .. code-block:: python
            from langchain.llms import DeepSparse
            llm = DeepSparse(model="zoo:nlg/text_generation/codegen_mono-350m/pytorch/huggingface/bigpython_bigquery_thepile/base_quant-none")
    """  # noqa: E501
    pipeline: Any  #: :meta private:
    model: str
    """The path to a model file or directory or the name of a SparseZoo model stub."""
    model_config: Optional[Dict[str, Any]] = None
    """Key word arguments passed to the pipeline construction.
    Common parameters are sequence_length, prompt_sequence_length"""
    generation_config: Union[None, str, Dict] = None
    """GenerationConfig dictionary consisting of parameters used to control
    sequences generated for each prompt. Common parameters are: