update

models/cpp_qwen2.py

@@ -128,6 +128,8 @@ class Qwen2Simulator(Simulator):
 
         self.user_start_tokens = self.tokenize("<|im_start|>user\n")
         self.assistant_start_tokens = self.tokenize("<|im_start|>assistant\n")
+        # self.llm.generate  .set_cache   .last_n_tokens_size  .reset  .ctx ._ctx
+        # self.llm.set_cache()
 
     def tokenize(self, text):
         return self.llm.tokenize(text.encode("utf-8"))
@@ -142,6 +144,13 @@ class Qwen2Simulator(Simulator):
         return tokens
 
     def generate(self, history, stream=True):
+        """
+        额外前向：remains 5 to forward "<|im_end|>\n<|im_start|>assistant\n"
+
+        :param history:
+        :param stream:
+        :return:
+        """
         if history[-1]['role'] in ["user"]:
             start_tokens = self.assistant_start_tokens
         elif history[-1]['role'] in ["assistant", "system"]:
@@ -162,8 +171,6 @@ class Qwen2Simulator(Simulator):
 
     def _stream_generate(self, input_ids):
         logger.info(f"generation_kwargs {self.generation_kwargs}")
-
-        # self.llm.generate  .set_cache   .last_n_tokens_size  .reset  .ctx ._ctx
         output = self.llm.create_completion(
             input_ids,
             stream=True,
@@ -190,6 +197,7 @@ if __name__ == "__main__":
         print(generated_text, generated_tokens)
 
     for i in range(3):
+        generated_tokens = bot.strip_stoptokens(generated_tokens)
         messages.append(
             {"role": "user" if i % 2 == 0 else "assistant", "content": generated_text, "tokens": generated_tokens})
         print("######## requesting", messages)