{ | |
"name": "CharacterTokenizer", | |
"char_ords": [ | |
3090, | |
355, | |
1139, | |
777, | |
1060, | |
183, | |
32, | |
1049, | |
1122, | |
1106, | |
8056, | |
947, | |
1026, | |
36, | |
191, | |
8051, | |
1058, | |
1103, | |
10004, | |
776, | |
8736, | |
8747, | |
90, | |
8592, | |
1116, | |
353, | |
9829, | |
324, | |
1097, | |
1066, | |
1513, | |
1032, | |
367, | |
1514, | |
23565, | |
1084, | |
1054, | |
896, | |
185, | |
345, | |
8222, | |
251, | |
1040, | |
1085, | |
1112, | |
1062, | |
44, | |
700, | |
8704, | |
33310, | |
1117, | |
1038, | |
12515, | |
40, | |
1104, | |
1031, | |
64, | |
1079, | |
943, | |
240, | |
72, | |
20869, | |
39, | |
1073, | |
352, | |
196, | |
299, | |
8059, | |
9794, | |
8203, | |
1025, | |
8249, | |
63, | |
80, | |
1141, | |
31639, | |
729, | |
966, | |
1169, | |
1050, | |
101, | |
246, | |
1046, | |
1089, | |
1076, | |
12476, | |
112, | |
1053, | |
8166, | |
87, | |
27475, | |
318, | |
47, | |
968, | |
61608, | |
49, | |
8016, | |
12384, | |
1512, | |
8183, | |
12364, | |
9472, | |
12523, | |
937, | |
21152, | |
61558, | |
184, | |
1081, | |
37, | |
19977, | |
10024, | |
1115, | |
513, | |
365, | |
20238, | |
7940, | |
1056, | |
231, | |
83, | |
768, | |
932, | |
972, | |
224, | |
9, | |
9656, | |
52, | |
331, | |
8541, | |
1199, | |
113, | |
8746, | |
26494, | |
33, | |
1503, | |
1217, | |
43, | |
38646, | |
941, | |
944, | |
8539, | |
926, | |
12459, | |
226, | |
1063, | |
1078, | |
172, | |
34, | |
970, | |
199, | |
8211, | |
128165, | |
9786, | |
1047, | |
731, | |
64257, | |
5, | |
204, | |
130, | |
55, | |
34892, | |
107, | |
1493, | |
189, | |
167, | |
12377, | |
12408, | |
181, | |
252, | |
22303, | |
920, | |
235, | |
8540, | |
8218, | |
1211, | |
12365, | |
95, | |
259, | |
973, | |
1111, | |
8212, | |
903, | |
202, | |
961, | |
58, | |
56, | |
250, | |
12298, | |
65, | |
933, | |
2, | |
1107, | |
379, | |
1241, | |
8057, | |
12473, | |
62, | |
347, | |
75, | |
1114, | |
1082, | |
178, | |
151, | |
915, | |
53, | |
9679, | |
82, | |
1092, | |
928, | |
8115, | |
33021, | |
929, | |
194, | |
269, | |
232, | |
150, | |
7985, | |
65281, | |
77, | |
10052, | |
116, | |
12418, | |
958, | |
942, | |
205, | |
73, | |
1099, | |
1105, | |
65039, | |
201, | |
195, | |
618, | |
111, | |
61482, | |
974, | |
977, | |
42, | |
1071, | |
1101, | |
274, | |
344, | |
727, | |
127, | |
8243, | |
245, | |
79, | |
938, | |
12540, | |
967, | |
1102, | |
957, | |
124, | |
12383, | |
8722, | |
12358, | |
1119, | |
268, | |
1055, | |
8730, | |
27498, | |
8194, | |
108, | |
242, | |
8453, | |
1088, | |
254, | |
12517, | |
122, | |
8221, | |
351, | |
234, | |
192, | |
945, | |
24231, | |
8594, | |
88, | |
10240, | |
948, | |
8060, | |
8017, | |
1067, | |
8216, | |
228, | |
328, | |
61449, | |
98, | |
279, | |
161, | |
99, | |
8226, | |
1497, | |
12463, | |
8150, | |
12417, | |
27468, | |
9825, | |
295, | |
119, | |
8531, | |
369, | |
221, | |
1091, | |
110, | |
173, | |
8195, | |
8596, | |
1029, | |
8197, | |
65279, | |
8049, | |
211, | |
106, | |
23383, | |
23376, | |
916, | |
160, | |
239, | |
28783, | |
19979, | |
114, | |
220, | |
103, | |
8164, | |
8729, | |
1187, | |
9824, | |
337, | |
1086, | |
8542, | |
8225, | |
9524, | |
188, | |
248, | |
8593, | |
8240, | |
118, | |
123, | |
1096, | |
923, | |
21475, | |
8741, | |
9633, | |
257, | |
12434, | |
180, | |
1036, | |
8364, | |
61620, | |
955, | |
197, | |
190, | |
8, | |
186, | |
244, | |
8470, | |
382, | |
10, | |
281, | |
1440, | |
8743, | |
380, | |
179, | |
8242, | |
230, | |
934, | |
363, | |
981, | |
833, | |
200, | |
12512, | |
8001, | |
96, | |
218, | |
8482, | |
1041, | |
41, | |
1095, | |
7990, | |
9658, | |
1068, | |
54, | |
25126, | |
84, | |
925, | |
12414, | |
70, | |
8776, | |
8532, | |
253, | |
960, | |
8055, | |
917, | |
236, | |
28779, | |
950, | |
57, | |
38, | |
81, | |
61448, | |
182, | |
935, | |
1140, | |
8058, | |
67, | |
1027, | |
215, | |
8322, | |
85, | |
1138, | |
1028, | |
271, | |
166, | |
1069, | |
946, | |
187, | |
7936, | |
177, | |
969, | |
8224, | |
171, | |
8190, | |
922, | |
12525, | |
105, | |
1144, | |
28792, | |
126, | |
338, | |
16, | |
65533, | |
7969, | |
1256, | |
965, | |
20307, | |
12395, | |
1061, | |
8052, | |
12424, | |
225, | |
25731, | |
12, | |
8486, | |
59, | |
1030, | |
61, | |
464, | |
12471, | |
287, | |
1042, | |
1502, | |
8800, | |
35, | |
1175, | |
1257, | |
733, | |
7937, | |
918, | |
255, | |
9632, | |
900, | |
8054, | |
76, | |
1087, | |
174, | |
1524, | |
50, | |
60, | |
8230, | |
48, | |
120, | |
964, | |
29226, | |
8038, | |
121, | |
8021, | |
1110, | |
1100, | |
1044, | |
1113, | |
971, | |
949, | |
963, | |
241, | |
263, | |
931, | |
1094, | |
17114, | |
237, | |
8220, | |
1045, | |
176, | |
9650, | |
1090, | |
378, | |
1080, | |
1033, | |
243, | |
65293, | |
36814, | |
1168, | |
69, | |
198, | |
914, | |
9830, | |
163, | |
1120, | |
12288, | |
953, | |
902, | |
1059, | |
8048, | |
212, | |
14, | |
1492, | |
321, | |
951, | |
904, | |
1072, | |
93, | |
227, | |
71, | |
1098, | |
952, | |
1093, | |
104, | |
1488, | |
7957, | |
1008, | |
74, | |
125, | |
216, | |
12426, | |
45, | |
1075, | |
7952, | |
9792, | |
1083, | |
38754, | |
277, | |
959, | |
1500, | |
46, | |
97, | |
12427, | |
956, | |
59393, | |
720, | |
305, | |
8202, | |
1052, | |
7883, | |
8213, | |
1074, | |
8125, | |
1118, | |
1043, | |
128519, | |
357, | |
1051, | |
8595, | |
233, | |
12360, | |
962, | |
665, | |
34241, | |
927, | |
12481, | |
1489, | |
89, | |
109, | |
210, | |
1057, | |
275, | |
92, | |
12507, | |
12356, | |
65288, | |
8239, | |
247, | |
710, | |
78, | |
1064, | |
261, | |
921, | |
164, | |
8165, | |
25239, | |
1123, | |
924, | |
8206, | |
769, | |
1039, | |
322, | |
115, | |
9574, | |
912, | |
66, | |
1108, | |
63719, | |
32946, | |
229, | |
8134, | |
117, | |
8287, | |
94, | |
1077, | |
940, | |
51, | |
214, | |
1070, | |
38651, | |
381, | |
913, | |
1065, | |
8020, | |
249, | |
91, | |
9827, | |
346, | |
8179, | |
1034, | |
86, | |
339, | |
350, | |
8209, | |
919, | |
100, | |
712, | |
8706, | |
1035, | |
223, | |
273, | |
1048, | |
954, | |
8196, | |
12398, | |
8182, | |
65289, | |
283, | |
1109, | |
238, | |
21322, | |
8805, | |
4, | |
8543, | |
68, | |
8210, | |
8217, | |
8250, | |
102, | |
169, | |
42012 | |
], | |
"model_max_length": 2048 | |
} |