Source code for langchain.llms.vllm
from typing import Any, Dict, List, Optional
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import BaseLLM
from langchain.llms.openai import BaseOpenAI
from langchain.pydantic_v1 import Field, root_validator
from langchain.schema.output import Generation, LLMResult
[docs]class VLLM(BaseLLM):
    """VLLM language model."""
    model: str = ""
    """The name or path of a HuggingFace Transformers model."""
    tensor_parallel_size: Optional[int] = 1
    """The number of GPUs to use for distributed execution with tensor parallelism."""
    trust_remote_code: Optional[bool] = False
    """Trust remote code (e.g., from HuggingFace) when downloading the model 
    and tokenizer."""
    n: int = 1
    """Number of output sequences to return for the given prompt."""
    best_of: Optional[int] = None
    """Number of output sequences that are generated from the prompt."""
    presence_penalty: float = 0.0
    """Float that penalizes new tokens based on whether they appear in the 
    generated text so far"""
    frequency_penalty: float = 0.0
    """Float that penalizes new tokens based on their frequency in the 
    generated text so far"""
    temperature: float = 1.0
    """Float that controls the randomness of the sampling."""
    top_p: float = 1.0
    """Float that controls the cumulative probability of the top tokens to consider."""
    top_k: int = -1
    """Integer that controls the number of top tokens to consider."""
    use_beam_search: bool = False
    """Whether to use beam search instead of sampling."""