charllama-35M / tokenizer_config.json
inkoziev's picture
Upload 3 files
1715cd7
raw
history blame
8.9 kB
{
"name": "CharacterTokenizer",
"char_ords": [
3090,
355,
1139,
777,
1060,
183,
32,
1049,
1122,
1106,
8056,
947,
1026,
36,
191,
8051,
1058,
1103,
10004,
776,
8736,
8747,
90,
8592,
1116,
353,
9829,
324,
1097,
1066,
1513,
1032,
367,
1514,
23565,
1084,
1054,
896,
185,
345,
8222,
251,
1040,
1085,
1112,
1062,
44,
700,
8704,
33310,
1117,
1038,
12515,
40,
1104,
1031,
64,
1079,
943,
240,
72,
20869,
39,
1073,
352,
196,
299,
8059,
9794,
8203,
1025,
8249,
63,
80,
1141,
31639,
729,
966,
1169,
1050,
101,
246,
1046,
1089,
1076,
12476,
112,
1053,
8166,
87,
27475,
318,
47,
968,
61608,
49,
8016,
12384,
1512,
8183,
12364,
9472,
12523,
937,
21152,
61558,
184,
1081,
37,
19977,
10024,
1115,
513,
365,
20238,
7940,
1056,
231,
83,
768,
932,
972,
224,
9,
9656,
52,
331,
8541,
1199,
113,
8746,
26494,
33,
1503,
1217,
43,
38646,
941,
944,
8539,
926,
12459,
226,
1063,
1078,
172,
34,
970,
199,
8211,
128165,
9786,
1047,
731,
64257,
5,
204,
130,
55,
34892,
107,
1493,
189,
167,
12377,
12408,
181,
252,
22303,
920,
235,
8540,
8218,
1211,
12365,
95,
259,
973,
1111,
8212,
903,
202,
961,
58,
56,
250,
12298,
65,
933,
2,
1107,
379,
1241,
8057,
12473,
62,
347,
75,
1114,
1082,
178,
151,
915,
53,
9679,
82,
1092,
928,
8115,
33021,
929,
194,
269,
232,
150,
7985,
65281,
77,
10052,
116,
12418,
958,
942,
205,
73,
1099,
1105,
65039,
201,
195,
618,
111,
61482,
974,
977,
42,
1071,
1101,
274,
344,
727,
127,
8243,
245,
79,
938,
12540,
967,
1102,
957,
124,
12383,
8722,
12358,
1119,
268,
1055,
8730,
27498,
8194,
108,
242,
8453,
1088,
254,
12517,
122,
8221,
351,
234,
192,
945,
24231,
8594,
88,
10240,
948,
8060,
8017,
1067,
8216,
228,
328,
61449,
98,
279,
161,
99,
8226,
1497,
12463,
8150,
12417,
27468,
9825,
295,
119,
8531,
369,
221,
1091,
110,
173,
8195,
8596,
1029,
8197,
65279,
8049,
211,
106,
23383,
23376,
916,
160,
239,
28783,
19979,
114,
220,
103,
8164,
8729,
1187,
9824,
337,
1086,
8542,
8225,
9524,
188,
248,
8593,
8240,
118,
123,
1096,
923,
21475,
8741,
9633,
257,
12434,
180,
1036,
8364,
61620,
955,
197,
190,
8,
186,
244,
8470,
382,
10,
281,
1440,
8743,
380,
179,
8242,
230,
934,
363,
981,
833,
200,
12512,
8001,
96,
218,
8482,
1041,
41,
1095,
7990,
9658,
1068,
54,
25126,
84,
925,
12414,
70,
8776,
8532,
253,
960,
8055,
917,
236,
28779,
950,
57,
38,
81,
61448,
182,
935,
1140,
8058,
67,
1027,
215,
8322,
85,
1138,
1028,
271,
166,
1069,
946,
187,
7936,
177,
969,
8224,
171,
8190,
922,
12525,
105,
1144,
28792,
126,
338,
16,
65533,
7969,
1256,
965,
20307,
12395,
1061,
8052,
12424,
225,
25731,
12,
8486,
59,
1030,
61,
464,
12471,
287,
1042,
1502,
8800,
35,
1175,
1257,
733,
7937,
918,
255,
9632,
900,
8054,
76,
1087,
174,
1524,
50,
60,
8230,
48,
120,
964,
29226,
8038,
121,
8021,
1110,
1100,
1044,
1113,
971,
949,
963,
241,
263,
931,
1094,
17114,
237,
8220,
1045,
176,
9650,
1090,
378,
1080,
1033,
243,
65293,
36814,
1168,
69,
198,
914,
9830,
163,
1120,
12288,
953,
902,
1059,
8048,
212,
14,
1492,
321,
951,
904,
1072,
93,
227,
71,
1098,
952,
1093,
104,
1488,
7957,
1008,
74,
125,
216,
12426,
45,
1075,
7952,
9792,
1083,
38754,
277,
959,
1500,
46,
97,
12427,
956,
59393,
720,
305,
8202,
1052,
7883,
8213,
1074,
8125,
1118,
1043,
128519,
357,
1051,
8595,
233,
12360,
962,
665,
34241,
927,
12481,
1489,
89,
109,
210,
1057,
275,
92,
12507,
12356,
65288,
8239,
247,
710,
78,
1064,
261,
921,
164,
8165,
25239,
1123,
924,
8206,
769,
1039,
322,
115,
9574,
912,
66,
1108,
63719,
32946,
229,
8134,
117,
8287,
94,
1077,
940,
51,
214,
1070,
38651,
381,
913,
1065,
8020,
249,
91,
9827,
346,
8179,
1034,
86,
339,
350,
8209,
919,
100,
712,
8706,
1035,
223,
273,
1048,
954,
8196,
12398,
8182,
65289,
283,
1109,
238,
21322,
8805,
4,
8543,
68,
8210,
8217,
8250,
102,
169,
42012
],
"model_max_length": 2048
}