refactor to packages, transformerschat

README.md

@@ -5,7 +5,7 @@ colorFrom: yellow
 colorTo: red
 sdk: streamlit
 sdk_version: 1.33.0
-app_file: app.py
+app_file: chat/app.py
 pinned: false
 ---
 

app.py

@@ -1,71 +1,4 @@
-import re
-
-import streamlit as st
-from loguru import logger
-# from ollamachat import ask, models
-from transformerschat import ask, models
-
-available_models = models()
-
-
-class Actor:
-    actors = {}
-
-    def __init__(self, role, model, system_prompt, pre_prompt):
-        self.role = role
-        self.model = model
-        self.system_prompt = system_prompt
-        self.pre_prompt = pre_prompt
-        Actor.actors[role] = self
-
-    def __class_getitem__(cls, item):
-        return cls.actors[item]
-
-
-def setup(question):
-    pp1 = pp2 = pp3 = "Ask the other two by always starting your sentence with their role. Never start your sentence with your own name. Share your inner thoughts inside parentheses. SAY ONLY ONE SINGLE SENTENCE!"
-    priest = Actor("Priest", available_models[0], "You are the Priest. There are 3 people standing in a circle: the Priest (that's you), the Teacher and the Kid.", pp1)
-    teacher = Actor("Teacher", available_models[0], "You are the Teacher. There are 3 people standing in a circle: the Priest, the Teacher (that's you) and the Kid.", pp2)
-    kid = Actor("Kid", available_models[0], "You are the Kid. There are 3 people standing in a circle: the Priest, the Teacher and the Kid (that's you).", pp3)
-    st.set_page_config(layout="wide")
-    col1, col2, col3 = st.columns(3)
-    for actor, col in [(priest, col1), (teacher, col2), (kid, col3)]:
-        with col:
-            role = actor.role
-            st.title(role)
-            actor.model = st.selectbox("model", available_models, key=f"{role}-model")
-            actor.system_prompt = st.text_area("system-prompt", actor.system_prompt, key=f"{role}-sp")
-            actor.pre_prompt = st.text_area("pre-prompt", actor.pre_prompt, key=f"{role}-pp")
-    st.text_input("Priest's task", f"{question}")
-    return question
-
-
-def main():
-    question = setup("Priest, your task is to figure out their names and where they live. Do not ask directly, they must not realize what information you are after!")
-
-    actor = target(sanitize(question))
-    max_steps = 1
-    for step, _ in enumerate(range(max_steps), start=1):
-        with st.spinner(f"({step}/{max_steps}) Asking {actor.role}..."):
-            answer = ask(actor.model, actor.system_prompt, actor.pre_prompt, question)
-            st.write(f":blue[{actor.role} says:] {answer}")
-            question = sanitize(answer)
-            actor = target(question)
-
-
-# noinspection PyTypeChecker
-def target(question) -> Actor:
-    try:
-        role = re.split(r'\s|,|:', question.strip())[0].strip()
-        return Actor[role]
-    except KeyError:
-        logger.warning(f"no actor found in question: {question}, trying to return the first actor")
-        return next(iter(Actor.actors.items()))[1]
-
-
-def sanitize(question):
-    return re.sub(r"\([^)]*\)", "", question)
-
+import chat
 
 if __name__ == "__main__":
-    main()
+    chat.main()

chat/__init__.py

@@ -0,0 +1,67 @@
+import re
+
+import streamlit as st
+from loguru import logger
+# from .ollamachat import ask, models
+from .transformerschat import ask, models
+
+available_models = models()
+
+
+class Actor:
+    actors = {}
+
+    def __init__(self, role, model, system_prompt, pre_prompt):
+        self.role = role
+        self.model = model
+        self.system_prompt = system_prompt
+        self.pre_prompt = pre_prompt
+        Actor.actors[role] = self
+
+    def __class_getitem__(cls, item):
+        return cls.actors[item]
+
+
+def setup(question):
+    pp1 = pp2 = pp3 = "Ask the other two by always starting your sentence with their role. Never start your sentence with your own name. Share your inner thoughts inside parentheses. SAY ONLY ONE SINGLE SENTENCE!"
+    priest = Actor("Priest", available_models[0], "You are the Priest. There are 3 people standing in a circle: the Priest (that's you), the Teacher and the Kid.", pp1)
+    teacher = Actor("Teacher", available_models[0], "You are the Teacher. There are 3 people standing in a circle: the Priest, the Teacher (that's you) and the Kid.", pp2)
+    kid = Actor("Kid", available_models[0], "You are the Kid. There are 3 people standing in a circle: the Priest, the Teacher and the Kid (that's you).", pp3)
+    st.set_page_config(layout="wide")
+    col1, col2, col3 = st.columns(3)
+    for actor, col in [(priest, col1), (teacher, col2), (kid, col3)]:
+        with col:
+            role = actor.role
+            st.title(role)
+            actor.model = st.selectbox("model", available_models, key=f"{role}-model")
+            actor.system_prompt = st.text_area("system-prompt", actor.system_prompt, key=f"{role}-sp")
+            actor.pre_prompt = st.text_area("pre-prompt", actor.pre_prompt, key=f"{role}-pp")
+    st.text_input("Priest's task", f"{question}")
+    return question
+
+
+def main():
+    question = setup("Priest, your task is to figure out their names and where they live. Do not ask directly, they must not realize what information you are after!")
+
+    actor = target(sanitize(question))
+    max_steps = 1
+    for step, _ in enumerate(range(max_steps), start=1):
+        with st.spinner(f"({step}/{max_steps}) Asking {actor.role}..."):
+            answer = ask(actor.model, actor.system_prompt, actor.pre_prompt, question)
+            st.write(f":blue[{actor.role} says:] {answer}")
+            question = sanitize(answer)
+            actor = target(question)
+
+
+# noinspection PyTypeChecker
+def target(question) -> Actor:
+    try:
+        role = re.split(r'\s|,|:', question.strip())[0].strip()
+        return Actor[role]
+    except KeyError:
+        logger.warning(f"no actor found in question: {question}, trying to return the first actor")
+        return next(iter(Actor.actors.items()))[1]
+
+
+def sanitize(question):
+    return re.sub(r"\([^)]*\)", "", question)

ollamachat.py → chat/ollamachat.py

@@ -8,14 +8,8 @@ def models():
 
 def ask(model, system_prompt, pre_prompt, question):
     messages = [
-        {
-            'role': 'system',
-            'content': f"{system_prompt} {pre_prompt}",
-        },
-        {
-            'role': 'user',
-            'content': f"{question}",
-        },
+        {'role': 'system', 'content': f"{system_prompt} {pre_prompt}", },
+        {'role': 'user', 'content': f"{question}", },
     ]
     logger.debug(f"<< {model} << {question}")
     response = ollama.chat(model=model, messages=messages)

chat/transformerschat.py

@@ -0,0 +1,48 @@
+from ctransformers import AutoModelForCausalLM, AutoTokenizer
+from loguru import logger
+import os
+
+
+def models():
+    return ["openhermes-2.5-mistral-7b.Q4_K_M.gguf"]
+
+
+def load():
+    # model = AutoModelForCausalLM.from_pretrained("TheBloke/OpenHermes-2.5-Mistral-7B-GGUF", model_file="openhermes-2.5-mistral-7b.Q4_K_M.gguf", model_type="mistral", gpu_layers=0, hf=True)
+
+    model = AutoModelForCausalLM.from_pretrained(
+        model_path_or_repo_id="TheBloke/Mistral-7B-OpenOrca-GGUF",
+        model_file="mistral-7b-openorca.Q5_K_M.gguf",
+        model_type="mistral",
+        hf=True,
+        temperature=0.7,
+        top_p=0.7,
+        top_k=50,
+        repetition_penalty=1.2,
+        context_length=32768,
+        max_new_tokens=2048,
+        threads=os.cpu_count(),
+        stream=True,
+        gpu_layers=0
+    )
+
+    tokenizer = AutoTokenizer.from_pretrained(model)
+    return (model, tokenizer)
+
+
+model, tokenizer = load()
+
+
+def ask(_, system_prompt, pre_prompt, question):
+    messages = [
+        {'role': 'system', 'content': f"{system_prompt} {pre_prompt}", },
+        {'role': 'user', 'content': f"{question}", },
+    ]
+    logger.debug(f"<< openhermes << {messages}")
+    # inputs = tokenizer.apply_chat_template(messages, return_tensors="pt")
+    inputs = tokenizer.apply_chat_template(messages, return_tensors="pt")
+
+    outputs = model.generate(inputs, max_length=200)
+    answer = tokenizer.batch_decode(outputs)[0]
+    logger.debug(f">> openhermes >> {answer}")
+    return answer

requirements.txt

@@ -3,6 +3,6 @@ streamlit
 ollama
 loguru
 pytest
-transformers
 torch
+spaces
 

test_sanitize.py

@@ -1,4 +1,4 @@
-from app import sanitize, target, Actor
+from app import sanitize, target
 
 
 def test_sanitize():

transformerschat.py

@@ -1,38 +0,0 @@
-import torch
-from ctransformers import AutoModelForCausalLM, AutoTokenizer
-from loguru import logger
-import spaces
-
-
-def models():
-    return ["openhermes-2.5-mistral-7b.Q4_K_M.gguf"]
-
-
-def load():
-    # torch.set_default_device("cuda")
-    model = AutoModelForCausalLM.from_pretrained("TheBloke/OpenHermes-2.5-Mistral-7B-GGUF", model_file="openhermes-2.5-mistral-7b.Q4_K_M.gguf", model_type="mistral", gpu_layers=50)
-    # tokenizer = AutoTokenizer.from_pretrained(models()[0], trust_remote_code=True).to("cuda")
-    return (model, tokenizer)
-
-
-model, tokenizer = load()
-
-
-def ask(_, system_prompt, pre_prompt, question):
-    messages = [
-        {
-            'role': 'system',
-            'content': f"{system_prompt} {pre_prompt}",
-        },
-        {
-            'role': 'user',
-            'content': f"{question}",
-        },
-    ]
-    logger.debug(f"<< openhermes << {question}")
-    # inputs = tokenizer(question, return_tensors="pt", return_attention_mask=False)
-    # outputs = model.generate(**inputs, max_length=200)
-    # answer = tokenizer.batch_decode(outputs)[0]
-    answer = model(question)
-    logger.debug(f">> openhermes >> {answer}")
-    return answer