feat: add stream

main.py

@@ -44,8 +44,8 @@ DB = Chroma(
 
 RETRIEVER = DB.as_retriever()
 
-LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME, stream=False)
-prompt, memory = get_prompt_template(promptTemplate_type="llama", history=False)
+LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME)
+prompt, memory = get_prompt_template(promptTemplate_type="llama", history=True)
 
 template = """you are a helpful, respectful and honest assistant.
 Your name is Katara llma. You should only use the source documents provided to answer the questions.
@@ -118,7 +118,7 @@ def run_ingest_route():
         )
 
         RETRIEVER = DB.as_retriever()
-        prompt, memory = get_prompt_template(promptTemplate_type="llama", history=False)
+        prompt, memory = get_prompt_template(promptTemplate_type="llama", history=True)
 
         QA = RetrievalQA.from_chain_type(
             llm=LLM,

run_localGPT.py

@@ -38,7 +38,7 @@ from constants import (
 
 
 
-def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stream=False):
+def load_model(device_type, model_id, model_basename=None, LOGGING=logging):
     """
     Select a model for text generation using the HuggingFace library.
     If you are running this for the first time, it will download a model for you.
@@ -56,7 +56,6 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stre
     Raises:
         ValueError: If an unsupported model or device type is provided.
     """
-    streamer = TextStreamer(tokenizer, skip_prompt=stream)
 
     logging.info(f"Loading Model: {model_id}, on: {device_type}")
     logging.info("This action can take a few minutes!")
@@ -80,6 +79,8 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stre
 
     # Create a pipeline for text generation
 
+    streamer = TextStreamer(tokenizer, skip_prompt=True)
+
     pipe = pipeline(
         "text-generation",
         model=model,

static/index.html

@@ -6,7 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <!-- Bootstrap CSS -->
 
-    <link rel="icon" type="image" href="static\social_icons\favicon.png" />
+    <!-- <link rel="icon" type="image" href="static\social_icons\favicon.png" /> -->
     <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
     <link
       href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css"
@@ -20,7 +20,7 @@
       crossorigin="anonymous"
     ></script>
 
-    <title>localGPT</title>
+    <title>Katara llama</title>
   </head>
   <style>
     @import url("https://fonts.googleapis.com/css2?family=Play:wght@400;700&display=swap");
@@ -56,7 +56,7 @@
       }
     </script>
     <div class="container">
-      <div class="titlegpt">KATARA LLMA</div>
+      <div class="titlegpt">Katara llama</div>
       <button
         type="button"
         style="