Update tokenization_rwkv_world.py

tokenization_rwkv_world.py

@@ -93,7 +93,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
     vocab_files_names = VOCAB_FILES_NAMES
     model_input_names = ["input_ids", "attention_mask"]
 
-    def __init__(self, vocab_file, errors="replace", pad_token="\n", **kwargs):
+    def __init__(self, vocab_file, errors="replace", pad_token="0", **kwargs):
         self.add_bos_token = False
         self.encoder = {}
         sorted = []  # must be already sorted
@@ -133,7 +133,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
 
     @property
     def pad_token_id(self) -> Optional[int]:
-        return 11
+        return 0
 
     @property
     def vocab_size(self):
@@ -316,7 +316,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
         verbose: bool = True,
         **kwargs,
     ) -> BatchEncoding:
-        def get_input_ids(text, max_length=None, pad_token_id=11):
+        def get_input_ids(text, max_length=None, pad_token_id=0):
             def pad_sequence(seq, max_len, pad_tok):
                 return [pad_tok] * (max_len - len(seq)) + seq