pszemraj
/

claude-tokenizer-mlm

Inference Endpoints

Model card Files Files and versions Community

pszemraj commited on Mar 14

Commit

83fe912

•

1 Parent(s): 66716bb

fix post_processor ids

Files changed (1) hide show

tokenizer.json +4 -4

tokenizer.json CHANGED Viewed

@@ -104,12 +104,12 @@
   "post_processor": {
     "type": "RobertaProcessing",
     "sep": [
-      "</s>",
-      2
     ],
     "cls": [
-      "<s>",
-      0
     ],
     "trim_offsets": true,
     "add_prefix_space": false

   "post_processor": {
     "type": "RobertaProcessing",
     "sep": [
+      "<EOT>",
+      0
     ],
     "cls": [
+      "<bos>",
+      65002
     ],
     "trim_offsets": true,
     "add_prefix_space": false