add tokenizer

added_tokens.json

@@ -1,3 +1 @@
-{
-  "[MASK]": 128000
-}
+{"[MASK]": 128000}

special_tokens_map.json

@@ -1,9 +1 @@
-{
-  "bos_token": "[CLS]",
-  "cls_token": "[CLS]",
-  "eos_token": "[SEP]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
-}
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -1,16 +1 @@
-{
-  "bos_token": "[CLS]",
-  "cls_token": "[CLS]",
-  "do_lower_case": false,
-  "eos_token": "[SEP]",
-  "mask_token": "[MASK]",
-  "name_or_path": "tner_ckpt/mit_movie_trivia_deberta_v3_large/model_rgwuwr/epoch_5",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "sp_model_kwargs": {},
-  "special_tokens_map_file": null,
-  "split_by_punct": false,
-  "tokenizer_class": "DebertaV2Tokenizer",
-  "unk_token": "[UNK]",
-  "vocab_type": "spm"
-}
+{"do_lower_case": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "split_by_punct": false, "sp_model_kwargs": {}, "name_or_path": "deberta-v3-large-mit-movie-trivia", "special_tokens_map_file": null, "vocab_type": "spm", "tokenizer_file": "deberta-v3-large-mit-movie-trivia/tokenizer.json", "tokenizer_class": "DebertaV2Tokenizer"}