Xenova
/

grok-1-tokenizer

Transformers.js

Inference Endpoints

Model card Files Files and versions Community

Update tokenizer_config.json

#2

by xianbao HF staff - opened Apr 19

base: refs/heads/main

←

from: refs/pr/2

Discussion Files changed

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -7,6 +7,6 @@
   "model_max_length": 8192,
   "pad_token": "[PAD]",
   "sp_model_kwargs": {},
-  "tokenizer_class": "Grok1Tokenizer",
   "unk_token": "[UNK]"
 }

   "model_max_length": 8192,
   "pad_token": "[PAD]",
   "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizerFast",
   "unk_token": "[UNK]"
 }