update

app.py

@@ -1,5 +1,6 @@
 """
 """
+import gradio
 
 import config
 from app_util import *
@@ -12,7 +13,7 @@ system_list = [
     "你是一个心理咨询师。",
 ]
 
-doc = """\
+user_simulator_doc = """\
 There are maily two types of user simulator:
 - prompt-based user-simulator (role-play)
 - model-based user-simulator
@@ -21,7 +22,6 @@ There are maily two types of user simulator:
 with gr.Blocks() as demo:
     # Knowledge Distillation through Self Chatting
     gr.HTML("""<h1 align="center">Distilling the Knowledge through Self Chatting</h1>""")
-    gr.Markdown(doc, visible=False)
     with gr.Row():
         with gr.Column(scale=5):
             system = gr.Dropdown(
@@ -33,18 +33,45 @@ with gr.Blocks() as demo:
                 scale=5,
             )
 
-            chatbot = gr.Chatbot(avatar_images=("assets/man.png", "assets/bot.png"))
+            chatbot = gr.Chatbot(show_copy_button=True,
+                                 show_share_button=True,
+                                 avatar_images=("assets/man.png", "assets/bot.png"))
 
-            generated_text = gr.Textbox(show_label=False, placeholder="...", lines=10, visible=False)
-            generate_btn = gr.Button("🤔️ Generate", variant="primary")
-            with gr.Row():
-                retry_btn = gr.Button("🔄  Regenerate", variant="secondary", size="sm", )
-                undo_btn = gr.Button("↩️ Undo", variant="secondary", size="sm", )
-                clear_btn = gr.Button("🗑️  Clear", variant="secondary", size="sm", )  # 🧹 Clear History (清除历史)
-                # stop_btn = gr.Button("停止生成", variant="stop", visible=False)
+            with gradio.Tab("Self Chat"):
+                generated_text_1 = gr.Textbox(show_label=False, placeholder="...", lines=10, visible=False)
+                generate_btn = gr.Button("🤔️ Self-Chat", variant="primary")
+                with gr.Row():
+                    retry_btn = gr.Button("🔄  Retry", variant="secondary", size="sm", )
+                    undo_btn = gr.Button("↩️ Undo", variant="secondary", size="sm", )
+                    clear_btn = gr.Button("🗑️  Clear", variant="secondary", size="sm", )  # 🧹 Clear History (清除历史)
+                    # stop_btn = gr.Button("停止生成", variant="stop", visible=False)
+                gr.Markdown(
+                    "Self-chat is a demo, which makes the model talk to itself. "
+                    "It is based on user simulator and response generator.",
+                    visible=True)
+
+            with gradio.Tab("Response Generator"):
+                with gr.Row():
+                    generated_text_2 = gr.Textbox(show_label=False, placeholder="Please type your input", scale=7)
+                    generate_btn_2 = gr.Button("Send", variant="primary")
+                with gr.Row():
+                    retry_btn_2 = gr.Button("🔄  Regenerate", variant="secondary", size="sm", )
+                    undo_btn_2 = gr.Button("↩️ Undo", variant="secondary", size="sm", )
+                    clear_btn_2 = gr.Button("🗑️  Clear", variant="secondary", size="sm", )  # 🧹 Clear History (清除历史)
+                gr.Markdown("Response simulator is the most commonly used chatbot.")
+
+            with gradio.Tab("User Simulator"):
+                with gr.Row():
+                    generated_text_3 = gr.Textbox(show_label=False, placeholder="Please type your response", scale=7)
+                    generate_btn_3 = gr.Button("Send", variant="primary")
+                with gr.Row():
+                    retry_btn_3 = gr.Button("🔄  Regenerate", variant="secondary", size="sm", )
+                    undo_btn_3 = gr.Button("↩️ Undo", variant="secondary", size="sm", )
+                    clear_btn_3 = gr.Button("🗑️  Clear", variant="secondary", size="sm", )  # 🧹 Clear History (清除历史)
+                gr.Markdown(user_simulator_doc)
 
         with gr.Column(variant="compact"):
-        # with gr.Column():
+            # with gr.Column():
             model = gr.Dropdown(
                 ["Qwen2-0.5B-Instruct", "llama3.1", "gemini"],
                 value="Qwen2-0.5B-Instruct",
@@ -73,18 +100,17 @@ with gr.Blocks() as demo:
                     label="Top-k",
                 )
 
-
     ########
     history = gr.State([{"role": "system", "content": system_list[0]}])
     system.change(reset_state, inputs=[system], outputs=[chatbot, history])
     clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history])
 
-    generate_btn.click(generate, [chatbot, history], outputs=[generated_text, chatbot, history],
+    generate_btn.click(generate, [chatbot, history], outputs=[generated_text_1, chatbot, history],
                        show_progress="full")
-    retry_btn.click(undo_generate, [chatbot, history], outputs=[generated_text, chatbot, history]) \
-        .then(generate, [chatbot, history], outputs=[generated_text, chatbot, history],
+    retry_btn.click(undo_generate, [chatbot, history], outputs=[generated_text_1, chatbot, history]) \
+        .then(generate, [chatbot, history], outputs=[generated_text_1, chatbot, history],
               show_progress="full")
-    undo_btn.click(undo_generate, [chatbot, history], outputs=[generated_text, chatbot, history])
+    undo_btn.click(undo_generate, [chatbot, history], outputs=[generated_text_1, chatbot, history])
 
     slider_max_tokens.change(set_max_tokens, inputs=[slider_max_tokens])
     slider_temperature.change(set_temperature, inputs=[slider_temperature])

models/cpp_qwen2.py

@@ -119,12 +119,15 @@ class Qwen2Simulator(Simulator):
 
     def _stream_generate(self, inputs):
         logger.info(f"generation_kwargs {self.generation_kwargs}")
+
+        # self.llm.generate  .set_cache   .last_n_tokens_size  .reset  .ctx ._ctx
         output = self.llm(
             inputs,
             stream=True,
             **self.generation_kwargs
         )
         generated_text = ""
+        # TODO: 检测finish reason，如果是length，则shift，并继续生成。
         for out in output:
             stream = copy.deepcopy(out)
             generated_text += stream["choices"][0]["text"]