Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +2 -2
tokenizer.json
CHANGED
@@ -991,7 +991,7 @@
|
|
991 |
{
|
992 |
"type": "Metaspace",
|
993 |
"replacement": "▁",
|
994 |
-
"
|
995 |
},
|
996 |
{
|
997 |
"type": "Split",
|
@@ -1104,7 +1104,7 @@
|
|
1104 |
{
|
1105 |
"type": "Metaspace",
|
1106 |
"replacement": "▁",
|
1107 |
-
"
|
1108 |
},
|
1109 |
{
|
1110 |
"type": "Strip",
|
|
|
991 |
{
|
992 |
"type": "Metaspace",
|
993 |
"replacement": "▁",
|
994 |
+
"prepend_scheme": "always"
|
995 |
},
|
996 |
{
|
997 |
"type": "Split",
|
|
|
1104 |
{
|
1105 |
"type": "Metaspace",
|
1106 |
"replacement": "▁",
|
1107 |
+
"prepend_scheme": "never"
|
1108 |
},
|
1109 |
{
|
1110 |
"type": "Strip",
|