orionweller
/

test-flex-gpt

Model card Files Files and versions Community

oweller2 commited on 24 days ago

Commit

3512ae1

•

1 Parent(s): 1866ba4

add eos token

Files changed (1) hide show

tokenizer_config.json +2 -1

tokenizer_config.json CHANGED Viewed

@@ -936,5 +936,6 @@
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
   "tokenizer_class": "PreTrainedTokenizerFast",
-  "unk_token": "[UNK]"
 }

   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
   "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]",
+  "eos_token": "[SEP]"
 }