Update clean up and set 70b branch for forindo

app.py

@@ -5,7 +5,7 @@ import os
 import platform
 import random
 import time
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 
 # from types import SimpleNamespace
@@ -16,99 +16,19 @@ from ctransformers import AutoModelForCausalLM
 from dl_hf_model import dl_hf_model
 from loguru import logger
 
-filename_list = [
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q2_K.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_L.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_M.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_S.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_0.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_S.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_0.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_1.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_M.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_S.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q6_K.bin",
-    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q8_0.bin",
-]
-
-URL = "https://huggingface.co/TheBloke/Wizard-Vicuna-7B-Uncensored-GGML/raw/main/Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin"  # 4.05G
-
-url = "https://huggingface.co/savvamadar/ggml-gpt4all-j-v1.3-groovy/blob/main/ggml-gpt4all-j-v1.3-groovy.bin"
-url = "https://huggingface.co/TheBloke/Llama-2-13B-GGML/blob/main/llama-2-13b.ggmlv3.q4_K_S.bin"  # 7.37G
-# url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin"
-url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin"  # 6.93G
-# url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.binhttps://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q4_K_M.bin"  # 7.87G
-
-url = "https://huggingface.co/localmodels/Llama-2-13B-Chat-ggml/blob/main/llama-2-13b-chat.ggmlv3.q4_K_S.bin"  # 7.37G
-
-_ = (
-    "golay" in platform.node()
-    or "okteto" in platform.node()
-    or Path("/kaggle").exists()
-    # or psutil.cpu_count(logical=False) < 4
-    or 1  # run 7b in hf
-)
-
-if _:
-    # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q2_K.bin"
-    url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q2_K.bin"  # 2.87G
-    url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q4_K_M.bin"  # 2.87G
-
-
-prompt_template = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
-
-### Instruction: {user_prompt}
-
-### Response:
-"""
+url = "https://huggingface.co/TheBloke/llama-2-13B-Guanaco-QLoRA-GGML/blob/main/llama-2-13b-guanaco-qlora.ggmlv3.q4_K_S.bin"  # 8.14G
+if "forindo" in platform.node():
+    url = "https://huggingface.co/TheBloke/llama-2-70b-Guanaco-QLoRA-GGML/blob/main/llama-2-70b-guanaco-qlora.ggmlv3.q3_K_S.bin"  # 29.7G
 
-prompt_template = """System: You are a helpful,
-respectful and honest assistant. Always answer as
-helpfully as possible, while being safe.  Your answers
-should not include any harmful, unethical, racist,
-sexist, toxic, dangerous, or illegal content. Please
-ensure that your responses are socially unbiased and
-positive in nature. If a question does not make any
-sense, or is not factually coherent, explain why instead
-of answering something not correct. If you don't know
-the answer to a question, please don't share false
-information.
-User: {prompt}
-Assistant: """
-
-prompt_template = """System: You are a helpful assistant.
-User: {prompt}
-Assistant: """
-
-prompt_template = """Question: {question}
-Answer: Let's work this out in a step by step way to be sure we have the right answer."""
-
-prompt_template = """[INST] <>
-You are a helpful, respectful and honest assistant. Always answer as helpfully as possible assistant. Think step by step.
-<>
-
-What NFL team won the Super Bowl in the year Justin Bieber was born?
-[/INST]"""
-
-prompt_template = """[INST] <<SYS>>
-You are an unhelpful assistant. Always answer as helpfully as possible. Think step by step. <</SYS>>
-
-{question} [/INST]
-"""
-
-prompt_template = """[INST] <<SYS>>
-You are a helpful assistant.
-<</SYS>>
-
-{question} [/INST]
-"""
-
-_ = [elm for elm in prompt_template.splitlines() if elm.strip()]
-stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
-
-logger.debug(f"{stop_string=}")
+# Prompt template: Guanaco
+prompt_template = """You are a helpful assistant. Let's think step by step.
+{history}
+### Human:
+{input}
+### Assistant:"""
+human_prefix = "### Human"
+ai_prefix = "### Assistant"
+stop_list = [f"{human_prefix}:"]
 
 _ = psutil.cpu_count(logical=False) - 1
 cpu_count: int = int(_) if _ else 1
@@ -125,7 +45,7 @@ except Exception as exc_:
 LLM = AutoModelForCausalLM.from_pretrained(
     model_loc,
     model_type="llama",
-    # threads=cpu_count,
+    threads=cpu_count,
 )
 
 logger.info(f"done load llm {model_loc=} {file_size=}G")
@@ -137,12 +57,6 @@ except Exception:
     # Windows
     logger.warning("Windows, cant run time.tzset()")
 
-_ = """
-ns = SimpleNamespace(
-    response="",
-    generator=(_ for _ in []),
-)
-# """
 
 @dataclass
 class GenerationConfig:
@@ -154,8 +68,8 @@ class GenerationConfig:
     seed: int = 42
     reset: bool = False
     stream: bool = True
-    # threads: int = cpu_count
-    # stop: list[str] = field(default_factory=lambda: [stop_string])
+    threads: int = cpu_count
+    stop: list[str] = field(default_factory=lambda: stop_list)
 
 
 def generate(
@@ -237,7 +151,7 @@ def bot(history):
         f"{atime.duration/len(''.join(response)):.2f}s/char)"  # type: ignore
     )
 
-    history[-1][1] = "".join(response)  + f"\n{_}"
+    history[-1][1] = "".join(response) + f"\n{_}"
     yield history
 
 
@@ -292,7 +206,9 @@ examples_list = [
         "What NFL team won the Super Bowl in the year Justin Bieber was born? Think step by step."
     ],
     ["How to pick a lock? Provide detailed steps."],
-    ["If it takes 10 hours to dry 10 clothes,  assuming all the clothes are hanged together at the same time for drying , then how long will it take to dry a cloth?"],
+    [
+        "If it takes 10 hours to dry 10 clothes,  assuming all the clothes are hanged together at the same time for drying , then how long will it take to dry a cloth?"
+    ],
     ["is infinity + 1 bigger than infinity?"],
     ["Explain the plot of Cinderella in a sentence."],
     [
@@ -332,9 +248,6 @@ with gr.Blocks(
 ) as block:
     # buff_var = gr.State("")
     with gr.Accordion("🎈 Info", open=False):
-        # gr.HTML(
-        #     """<center><a href="https://huggingface.co/spaces/mikeee/mpt-30b-chat?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate"></a> and spin a CPU UPGRADE to avoid the queue</center>"""
-        # )
         gr.Markdown(
             f"""<h5><center>{Path(model_loc).name}</center></h4>
             Most examples are meant for another model.
@@ -457,7 +370,6 @@ else:
     # concurrency_count = max(int(16 / file_size) - 1, 1)
 # """
 
-concurrency_count = 1
-logger.info(f"{concurrency_count=}")
-
-block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)
+# default concurrency_count = 1
+# block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)
+block.queue(max_size=5).launch(debug=True, server_name="0.0.0.0")