update

app.py

@@ -4,15 +4,13 @@
 
 # 难点
 
-
-
 ## TODO
 
-- 第一句：
-- 代码和表格的预览
-- markdown解析：mdtex2html
-- 可编辑chatbot：https://github.com/gradio-app/gradio/issues/4444
-- 一个button，
+
+-[x] 代码和表格的预览
+-[x] markdown解析：mdtex2html
+-[ ] 可编辑chatbot：https://github.com/gradio-app/gradio/issues/4444
+-[ ] 乱码问题
 
 
 ## Reference
@@ -20,6 +18,7 @@
 - https://github.com/GaiZhenbiao/ChuanhuChatGPT/
 """
 
+import config
 from app_util import *
 
 system_list = [
@@ -58,38 +57,42 @@ with gr.Blocks() as demo:
             gr.Dropdown(
                 ["moss", "chatglm-2", "chatpdf"],
                 value="moss",
-                label="问题生成器",
-                # info="Will add more animals later!"
-            )
-            gr.Dropdown(
-                ["moss", "chatglm-2", "gpt3.5-turbo"],
-                value="gpt3.5-turbo",
-                label="回复生成器",
+                label="model",
+                interactive=True,
                 # info="Will add more animals later!"
             )
 
-        slider_max_new_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
-        slider_temperature = gr.Slider(minimum=0.1, maximum=10.0, value=5, step=0.1, label="Temperature",
+        slider_max_tokens = gr.Slider(minimum=1, maximum=config.MAX_SEQUENCE_LENGTH,
+                                      value=config.DEFAULT_MAX_TOKENS, step=1, label="Max tokens")
+        slider_temperature = gr.Slider(minimum=0.1, maximum=10.0,
+                                       value=config.DEFAULT_TEMPERATURE, step=0.1, label="Temperature",
                                        info="Larger temperature increase the randomness")
         slider_top_p = gr.Slider(
             minimum=0.1,
             maximum=1.0,
-            value=0.95,
+            value=config.DEFAULT_TOP_P,
             step=0.05,
             label="Top-p (nucleus sampling)",
         )
 
     ########
     history = gr.State([{"role": "system", "content": system_list[0]}])
-    system.change(reset_state, inputs=[system], outputs=[chatbot, history], show_progress="full")
-    # submit_btn.click(reset_user_input, [], [user_input])
-
-    clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history], show_progress="full")
+    system.change(reset_state, inputs=[system], outputs=[chatbot, history])
+    clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history])
 
     generate_btn.click(generate, [chatbot, history], outputs=[generated_text, chatbot, history],
                        show_progress="full")
-
-    slider_max_new_tokens.change(set_max_tokens, inputs=[slider_max_new_tokens])
-
-demo.queue().launch(share=False, server_name="0.0.0.0")
-# demo.queue().launch(share=True)
+    retry_btn.click(undo_generate, [chatbot, history], outputs=[generated_text, chatbot, history],
+                    show_progress="full")
+    retry_btn.click(generate, [chatbot, history], outputs=[generated_text, chatbot, history],
+                    show_progress="full")
+    undo_btn.click(undo_generate, [chatbot, history], outputs=[generated_text, chatbot, history],
+                   show_progress="full")
+
+    slider_max_tokens.change(set_max_tokens, inputs=[slider_max_tokens])
+    slider_top_p.change(set_top_p, inputs=[slider_top_p])
+    slider_temperature.change(set_temperature, inputs=[slider_temperature])
+
+# demo.queue().launch(share=False, server_name="0.0.0.0")
+# demo.queue().launch(concurrency_count=1, max_size=5)
+demo.queue().launch()

app_util.py

@@ -1,11 +1,8 @@
-
 import gradio as gr
 from utils.logging_util import logger
 from models.cpp_qwen2 import bot
 
 
-
-
 #
 # def postprocess(self, y):
 #     if y is None:
@@ -95,11 +92,21 @@ def set_max_tokens(max_tokens):
     bot.generation_kwargs["max_tokens"] = max_tokens
 
 
-def clear_history():
-    pass
+def set_top_p(top_p):
+    bot.generation_kwargs["top_p"] = top_p
 
 
-def undo_generate():
-    pass
+def set_temperature(temperature):
+    bot.generation_kwargs["temperature"] = temperature
 
 
+def undo_generate(chatbot, history):
+    if history[-1]["role"] == "user":
+        history = history[:-1]
+        chatbot = chatbot[:-1]
+    elif history[-1]["role"] == "assistant":
+        history = history[:-1]
+        chatbot[-1] = (chatbot[-1][0], None)
+    else:
+        pass
+    return "", chatbot, history

config.py

@@ -0,0 +1,9 @@
+
+
+MAX_SEQUENCE_LENGTH = 2048  # max_seq_len
+
+DEFAULT_MAX_TOKENS = 512
+# DEFAULT_MAX_NEW_TOKENS = None
+DEFAULT_TOP_K = 100
+DEFAULT_TOP_P = 0.95
+DEFAULT_TEMPERATURE = 5

models/cpp_qwen2.py

@@ -17,11 +17,12 @@ python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen1.5-0.5B-C
 
 import json
 import copy
-from simulator import Simulator
+from base_model import Simulator
 import llama_cpp
 # import llama_cpp.llama_tokenizer
 from transformers import AutoTokenizer
 from utils.logging_util import logger
+import config
 
 
 class Qwen2Simulator(Simulator):
@@ -30,9 +31,12 @@ class Qwen2Simulator(Simulator):
         if from_local:
             self.hf_tokenizer = AutoTokenizer.from_pretrained(
                 "/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/")
-            self.llm = llama_cpp.Llama(
+            self.llm = llama_cpp.Llama(  # n_ctx, n_threads
+
                 model_path="/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct-GGUF/qwen2-0_5b-instruct-fp16.gguf",
                 tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
+                n_ctx=config.MAX_SEQUENCE_LENGTH,  #
+                # n_threads=None, # 默认会根据cpu数来设置 n_threads
                 verbose=False,
             )
         else:
@@ -43,13 +47,13 @@ class Qwen2Simulator(Simulator):
                 tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
                 verbose=False,
             )
-        logger.info(f"llm has been initialized: {self.llm}")
+        logger.info(f"llm has been initialized: {self.llm}, n_threads={self.llm.n_threads}, n_ctx={self.llm.n_ctx}")
 
         self.generation_kwargs = dict(
-            temperature=5,
-            # top_p=0.1,
-            top_k=40,
-            max_tokens=20,
+            temperature=config.DEFAULT_TEMPERATURE,
+            top_p=config.DEFAULT_TOP_P,
+            top_k=config.DEFAULT_TOP_K,
+            max_tokens=config.DEFAULT_MAX_TOKENS,
             repeat_penalty=1.1,
             # qwen2-0.5b-chat 有时内容生成结束没有<|im_end|>，直接跟 <|im_start|>
             stop=[
@@ -58,7 +62,6 @@ class Qwen2Simulator(Simulator):
                 "<|endoftext|>",
             ],
         )
-        ### local
 
     def generate_query(self, messages, stream=True):
         """
@@ -66,7 +69,7 @@ class Qwen2Simulator(Simulator):
         :return:
         """
         assert messages[-1]["role"] != "user"
-        logger.info(f"generating {json.dumps(messages)}")
+        logger.info(f"generating {json.dumps(messages, ensure_ascii=False)}")
         inputs = self.hf_tokenizer.apply_chat_template(
             messages,
             tokenize=False,
@@ -78,7 +81,6 @@ class Qwen2Simulator(Simulator):
         else:
             return self._generate(inputs)
 
-
     def generate_response(self, messages, stream=True):
         assert messages[-1]["role"] == "user"
         logger.info(f"generating {json.dumps(messages, ensure_ascii=False)}")
@@ -93,10 +95,6 @@ class Qwen2Simulator(Simulator):
             return self._generate(inputs)
 
     def _generate(self, inputs):
-        """
-        TODO: chat with cache.
-
-        """
         logger.info(f"generation_kwargs {self.generation_kwargs}")
         output = self.llm(
             inputs,
@@ -117,6 +115,7 @@ class Qwen2Simulator(Simulator):
             generated_text += stream["choices"][0]["text"]
             yield generated_text
 
+
 bot = Qwen2Simulator()
 
 if __name__ == "__main__":

models/hf_qwen2.py

@@ -1,7 +1,7 @@
 "Qwen/Qwen2-0.5B-Instruct"
 
 from threading import Thread
-from simulator import Simulator
+from base_model import Simulator
 
 from transformers import TextIteratorStreamer
 from transformers import AutoModelForCausalLM, AutoTokenizer