Source code for langchain.llms.textgen
import json
import logging
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional
import requests
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.llms.base import LLM
from langchain.pydantic_v1 import Field
from langchain.schema.output import GenerationChunk
logger = logging.getLogger(__name__)
[docs]class TextGen(LLM):
    """text-generation-webui models.
    To use, you should have the text-generation-webui installed, a model loaded,
    and --api added as a command-line option.
    Suggested installation, use one-click installer for your OS:
    https://github.com/oobabooga/text-generation-webui#one-click-installers
    Parameters below taken from text-generation-webui api example:
    https://github.com/oobabooga/text-generation-webui/blob/main/api-examples/api-example.py
    Example:
        .. code-block:: python
            from langchain.llms import TextGen
            llm = TextGen(model_url="http://localhost:8500")
    """
    model_url: str
    """The full URL to the textgen webui including http[s]://host:port """
    preset: Optional[str] = None
    """The preset to use in the textgen webui """
    max_new_tokens: Optional[int] = 250
    """The maximum number of tokens to generate."""
    do_sample: bool = Field(True, alias="do_sample")
    """Do sample"""
    temperature: Optional[float] = 1.3
    """Primary factor to control randomness of outputs. 0 = deterministic
    (only the most likely token is used). Higher value = more randomness."""