update

app_util.py

@@ -29,6 +29,7 @@ def generate_query(chatbot, history):
         for query, query_tokens in streamer:
             chatbot[-1] = (query, None)
             yield query, chatbot, history
+        query_tokens = bot.strip_stoptokens(query_tokens)
         history.append({"role": "user", "content": query, "tokens": query_tokens})
         yield query, chatbot, history
 
@@ -49,6 +50,7 @@ def generate_response(chatbot, history):
             chatbot[-1] = (query, response)
             yield response, chatbot, history
 
+        response_tokens = bot.strip_stoptokens(response_tokens)
         history.append({"role": "assistant", "content": response, "tokens": response_tokens})
         print(f"chatbot is {chatbot}")
         print(f"history is {history}")

models/cpp_qwen2.py

@@ -86,7 +86,7 @@ class Qwen2Simulator(Simulator):
     def tokenize(self, text):
         return self.llm.tokenize(text.encode("utf-8"))
 
-    def _strip_stoptokens(self, tokens):
+    def strip_stoptokens(self, tokens):
         while tokens and tokens[0] in self.stop_tokens:
             tokens.pop(0)
         while tokens and tokens[-1] in self.stop_tokens:
@@ -104,7 +104,7 @@ class Qwen2Simulator(Simulator):
             if "tokens" not in message:  # tokens
                 message["tokens"] = self.tokenize(message["content"])
             input_ids += self.tokenize(f"<|im_start|>{message['role']}\n") \
-                         + self._strip_stoptokens(message["tokens"]) \
+                         + message["tokens"] \
                          + self.tokenize("<|im_end|>\n")
         input_ids += start_tokens
         if stream: