Upload tokenizer
Browse files- added_tokens.json +1 -2
- vocab.json +1 -1
added_tokens.json
CHANGED
|
@@ -1,5 +1,4 @@
|
|
| 1 |
{
|
| 2 |
"</s>": 49,
|
| 3 |
-
"<s>": 48
|
| 4 |
-
"[PAD]": 47
|
| 5 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"</s>": 49,
|
| 3 |
+
"<s>": 48
|
|
|
|
| 4 |
}
|
vocab.json
CHANGED
|
@@ -19,7 +19,7 @@
|
|
| 19 |
"S": 18,
|
| 20 |
"[PAD]": 47,
|
| 21 |
"[UNK]": 46,
|
| 22 |
-
"_":
|
| 23 |
"a": 20,
|
| 24 |
"b": 21,
|
| 25 |
"c": 22,
|
|
|
|
| 19 |
"S": 18,
|
| 20 |
"[PAD]": 47,
|
| 21 |
"[UNK]": 46,
|
| 22 |
+
"_": 19,
|
| 23 |
"a": 20,
|
| 24 |
"b": 21,
|
| 25 |
"c": 22,
|