fix tokenizer

special_tokens_map.json

@@ -12,5 +12,12 @@
     "normalized": false,
     "rstrip": false,
     "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
   }
 }

tokenizer.json

@@ -2300,7 +2300,7 @@
     },
     {
       "id": 128255,
-      "content": "<|reserved_special_token_250|>",
+      "content": "<pad>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2329,69 +2329,10 @@
     ]
   },
   "post_processor": {
-    "type": "Sequence",
-    "processors": [
-      {
-        "type": "ByteLevel",
-        "add_prefix_space": true,
-        "trim_offsets": false,
-        "use_regex": true
-      },
-      {
-        "type": "TemplateProcessing",
-        "single": [
-          {
-            "SpecialToken": {
-              "id": "<|begin_of_text|>",
-              "type_id": 0
-            }
-          },
-          {
-            "Sequence": {
-              "id": "A",
-              "type_id": 0
-            }
-          }
-        ],
-        "pair": [
-          {
-            "SpecialToken": {
-              "id": "<|begin_of_text|>",
-              "type_id": 0
-            }
-          },
-          {
-            "Sequence": {
-              "id": "A",
-              "type_id": 0
-            }
-          },
-          {
-            "SpecialToken": {
-              "id": "<|begin_of_text|>",
-              "type_id": 1
-            }
-          },
-          {
-            "Sequence": {
-              "id": "B",
-              "type_id": 1
-            }
-          }
-        ],
-        "special_tokens": {
-          "<|begin_of_text|>": {
-            "id": "<|begin_of_text|>",
-            "ids": [
-              128000
-            ],
-            "tokens": [
-              "<|begin_of_text|>"
-            ]
-          }
-        }
-      }
-    ]
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
   },
   "decoder": {
     "type": "ByteLevel",
@@ -2407,7 +2348,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,
-    "ignore_merges": true,
+    "ignore_merges": false,
     "vocab": {
       "!": 0,
       "\"": 1,
@@ -410560,4 +410501,4 @@
       "éĶ ¦"
     ]
   }
-}
+}

tokenizer_config.json

@@ -2041,7 +2041,7 @@
       "special": true
     },
     "128255": {
-      "content": "<|reserved_special_token_250|>",
+      "content": "<pad>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2050,6 +2050,7 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|end_of_text|>",
   "model_input_names": [
@@ -2057,6 +2058,6 @@
     "attention_mask"
   ],
   "model_max_length": 1000000000000000019884624838656,
-  "tokenizer_class": "PreTrainedTokenizerFast",
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}"
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
 }