aframson
/

RDPD-mini

Text Generation

feature-extraction

Model card Files Files and versions Community

drat

#1

by aframson - opened Sep 16, 2023

base: refs/heads/main

←

from: refs/pr/1

Discussion Files changed

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -2,7 +2,7 @@
   "tokenizer_type": "custom",
   "vocab_size": 92656,
   "tokenizer_class": "OBITokenizer",
-  "auto_map": { "AutoTokenizer": ["tokenizeConfig.OBITokenizer"] },
   "bos_token": "[CLS]",
   "eos_token": "[SEP]",
   "unk_token": "[UNK]",

   "tokenizer_type": "custom",
   "vocab_size": 92656,
   "tokenizer_class": "OBITokenizer",
+  "auto_map": { "AutoTokenizer": ["tokenizeConfig.OBITokenizer",null] },
   "bos_token": "[CLS]",
   "eos_token": "[SEP]",
   "unk_token": "[UNK]",