add kwargs

tokenization_rex_qwen2.py

@@ -41,17 +41,17 @@ class RexQwen2Tokenizer(Qwen2Tokenizer):
 
 
     # redefine the _tokenize method
-    def tokenize(self, text):   
+    def tokenize(self, text, **kwargs):   
         # find the index for first user query 
         if self.user_prefix not in text or self.rex_size < 1:
             # the query is not wrapped with chat template yet 
             # raise NotImplementedError
-            return super().tokenize(text)
+            return super().tokenize(text, **kwargs)
         start_index = text.index(self.user_prefix)
         rex_chat_history_tokens = self._rex_query(text[start_index+len(self.user_prefix):])
         rex_text = text[:start_index] + rex_chat_history_tokens + text[start_index:]
         # print(rex_text)
-        tokens = super().tokenize(rex_text)
+        tokens = super().tokenize(rex_text, **kwargs)
         return tokens