HPLT
/

hplt_bert_base_en

Model card Files Files and versions Community

davda54 commited on Sep 12

Commit

bd75dd5

•

1 Parent(s): 1108d9b

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +3 -5

tokenizer.json CHANGED Viewed

@@ -1102,11 +1102,9 @@
         "use_regex": true
       },
       {
-        "type": "Replace",
-        "pattern": {
-          "Regex": "▁"
-        },
-        "content": " "
       },
       {
         "type": "Strip",

         "use_regex": true
       },
       {
+        "type": "Metaspace",
+        "replacement": "▁",
+        "add_prefix_space": false
       },
       {
         "type": "Strip",