kmhf
/

hf-moshiko

ylacombe HF staff commited on Oct 4

Commit

652e33e

•

1 Parent(s): 569de4c

Upload tokenizer

Files changed (1) hide show

tokenizer.json CHANGED Viewed

@@ -56,7 +56,7 @@
     "type": "Metaspace",
     "replacement": "▁",
     "prepend_scheme": "first",
-    "split": true
   },
   "post_processor": null,
   "decoder": {

     "type": "Metaspace",
     "replacement": "▁",
     "prepend_scheme": "first",
+    "split": false
   },
   "post_processor": null,
   "decoder": {