change padding token to newline (id=11)

tokenization_rwkv_world.py

@@ -93,7 +93,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
     vocab_files_names = VOCAB_FILES_NAMES
     model_input_names = ["input_ids", "attention_mask"]
 
-    def __init__(self, vocab_file, errors="replace", pad_token="0", **kwargs):
+    def __init__(self, vocab_file, errors="replace", pad_token="\n", **kwargs):
         self.add_bos_token = False
         self.encoder = {}
         sorted = []  # must be already sorted
@@ -133,7 +133,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
 
     @property
     def pad_token_id(self) -> Optional[int]:
-        return 0
+        return 11
 
     @property
     def vocab_size(self):
@@ -264,11 +264,11 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
 
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
-        return self.encoder.get(token, self.encoder.get(self.unk_token))
+        return self.decoder.get(token.encode("utf-8"), self.unk_token_id)
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
-        return self.decoder.get(index)
+        return self.encoder.get(index)
 
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
         if not os.path.exists(save_directory):
@@ -316,7 +316,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
         verbose: bool = True,
         **kwargs,
     ) -> BatchEncoding:
-        def get_input_ids(text, max_length=None, pad_token_id=0):
+        def get_input_ids(text, max_length=None, pad_token_id=11):
             def pad_sequence(seq, max_len, pad_tok):
                 return [pad_tok] * (max_len - len(seq)) + seq