param logits_all: bool = False¶
Return logits for all tokens, not just the last token.
param logprobs: Optional[int] = None¶
The number of logprobs to return. If None, no logprobs are returned.
param lora_base: Optional[str] = None¶
The path to the Llama LoRA base model.
param lora_path: Optional[str] = None¶
The path to the Llama LoRA. If None, no LoRa is loaded.
param max_tokens: Optional[int] = 256¶
The maximum number of tokens to generate.
param metadata: Optional[Dict[str, Any]] = None¶
Metadata to add to the run trace.
param model_kwargs: Dict[str, Any] [Optional]¶
Any additional parameters to pass to llama_cpp.Llama.
param model_path: str [Required]¶
The path to the Llama model file.
param n_batch: Optional[int] = 8¶
Number of tokens to process in parallel.
Should be a number between 1 and n_ctx.
param n_ctx: int = 512¶
Token context window.
param n_gpu_layers: Optional[int] = None¶
Number of layers to be loaded into gpu memory. Default None.
param n_parts: int = -1¶
Number of parts to split the model into.
If -1, the number of parts is automatically determined.
param n_threads: Optional[int] = None¶
Number of threads to use.
If None, the number of threads is automatically determined.
param repeat_penalty: Optional[float] = 1.1¶
The penalty to apply to repeated tokens.
param rope_freq_base: float = 10000.0¶
Base frequency for rope sampling.
param rope_freq_scale: float = 1.0¶
Scale factor for rope sampling.