danielhanchen
commited on
Commit
•
021b26d
1
Parent(s):
ce7dabd
Upload tokenizer
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +2 -2
tokenizer.json
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3f289bc05132635a8bc7aca7aa21255efd5e18f3710f43e3cdb96bcd41be4922
|
3 |
+
size 17525357
|
tokenizer_config.json
CHANGED
@@ -856,7 +856,7 @@
|
|
856 |
"normalized": false,
|
857 |
"rstrip": false,
|
858 |
"single_word": false,
|
859 |
-
"special":
|
860 |
},
|
861 |
"107": {
|
862 |
"content": "<end_of_turn>",
|
@@ -864,7 +864,7 @@
|
|
864 |
"normalized": false,
|
865 |
"rstrip": false,
|
866 |
"single_word": false,
|
867 |
-
"special":
|
868 |
},
|
869 |
"108": {
|
870 |
"content": "\n",
|
|
|
856 |
"normalized": false,
|
857 |
"rstrip": false,
|
858 |
"single_word": false,
|
859 |
+
"special": true
|
860 |
},
|
861 |
"107": {
|
862 |
"content": "<end_of_turn>",
|
|
|
864 |
"normalized": false,
|
865 |
"rstrip": false,
|
866 |
"single_word": false,
|
867 |
+
"special": true
|
868 |
},
|
869 |
"108": {
|
870 |
"content": "\n",
|