Create model.py

model.py

@@ -0,0 +1,28 @@
+from threading import Thread
+from typing import Iterator
+
+import torch
+
+
+model_id = 'xuqinyang/baichuan-13b-chat-ggml-int4'
+
+from huggingface_hub import snapshot_download
+
+snapshot_download(model_id, local_dir=".\")
+from llama_cpp import Llama
+self.llm = Llama(model_path="./ggml-model-q4_0.bin", n_ctx=4096,seed=-1,n_threads=4)
+
+def run(message: str,
+        chat_history: list[tuple[str, str]],
+        system_prompt: str,
+        max_new_tokens: int = 1024,
+        temperature: float = 0.8,
+        top_p: float = 0.95,
+        top_k: int = 50) -> Iterator[str]:
+    history = []
+    for i in chat_history:
+        history.append({"role": "user", "content": i[0]})
+        history.append({"role": "assistant", "content": i[1]})
+    for response in self.llm.create_chat_completion(messages,stream=True,max_tokens=-1,temperature=0.3,top_k=5,top_p=0.85,repeat_penalty=1.1):
+        if "content" in response["choices"][0]["delta"]:
+            yield response["choices"][0]["delta"]["content"]