Source code for langchain.llms.ctranslate2
from typing import Any, Dict, List, Optional, Union
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import BaseLLM
from langchain.pydantic_v1 import Field, root_validator
from langchain.schema.output import Generation, LLMResult
[docs]class CTranslate2(BaseLLM):
    """CTranslate2 language model."""
    model_path: str = ""
    """Path to the CTranslate2 model directory."""
    tokenizer_name: str = ""
    """Name of the original Hugging Face model needed to load the proper tokenizer."""
    device: str = "cpu"
    """Device to use (possible values are: cpu, cuda, auto)."""
    device_index: Union[int, List[int]] = 0
    """Device IDs where to place this generator on."""
    compute_type: Union[str, Dict[str, str]] = "default"
    """
    Model computation type or a dictionary mapping a device name to the computation type
    (possible values are: default, auto, int8, int8_float32, int8_float16,
    int8_bfloat16, int16, float16, bfloat16, float32).
    """
    max_length: int = 512
    """Maximum generation length."""
    sampling_topk: int = 1
    """Randomly sample predictions from the top K candidates."""
    sampling_topp: float = 1
    """Keep the most probable tokens whose cumulative probability exceeds this value."""
    sampling_temperature: float = 1
    """Sampling temperature to generate more random samples."""
    client: Any  #: :meta private:
    tokenizer: Any  #: :meta private: