huseinzol05 commited on
Commit
d310b32
1 Parent(s): f7b14e8

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +87 -87
  2. tokenizer.json +99 -99
  3. tokenizer_config.json +186 -186
special_tokens_map.json CHANGED
@@ -1,105 +1,105 @@
1
  {
2
  "additional_special_tokens": [
3
- "<extra_id_39>",
4
- "<extra_id_95>",
5
- "<extra_id_93>",
6
- "<extra_id_79>",
7
- "<extra_id_32>",
8
- "<extra_id_15>",
9
- "<extra_id_73>",
10
- "<extra_id_36>",
11
  "<extra_id_1>",
12
- "<extra_id_19>",
13
- "<extra_id_26>",
14
- "<extra_id_60>",
15
- "<extra_id_51>",
16
- "<extra_id_5>",
17
  "<extra_id_3>",
18
- "<extra_id_31>",
19
- "<extra_id_58>",
20
- "<extra_id_23>",
21
- "<extra_id_50>",
22
- "<extra_id_78>",
23
- "<extra_id_96>",
24
- "<extra_id_72>",
25
- "<extra_id_74>",
26
  "<extra_id_4>",
27
- "<extra_id_48>",
28
- "<extra_id_42>",
29
- "<extra_id_90>",
30
- "<extra_id_82>",
31
- "<extra_id_57>",
32
- "<extra_id_34>",
33
- "<extra_id_16>",
34
- "<extra_id_33>",
35
- "<extra_id_35>",
36
- "<extra_id_69>",
37
  "<extra_id_14>",
38
- "<extra_id_65>",
39
- "<extra_id_88>",
40
- "<extra_id_97>",
41
- "<extra_id_94>",
42
- "<extra_id_27>",
43
- "<extra_id_80>",
44
- "<extra_id_41>",
45
- "<extra_id_55>",
46
- "<extra_id_68>",
47
- "<extra_id_98>",
48
- "<extra_id_24>",
49
  "<extra_id_22>",
50
- "<extra_id_49>",
51
- "<extra_id_91>",
52
- "<extra_id_64>",
53
- "<extra_id_99>",
54
- "<extra_id_13>",
55
- "<extra_id_45>",
56
  "<extra_id_29>",
57
- "<extra_id_11>",
58
- "<extra_id_87>",
59
- "<extra_id_47>",
 
 
 
 
60
  "<extra_id_37>",
61
- "<extra_id_86>",
62
- "<extra_id_63>",
 
 
 
63
  "<extra_id_43>",
64
- "<extra_id_25>",
65
- "<extra_id_21>",
66
- "<extra_id_7>",
67
- "<extra_id_92>",
68
- "<extra_id_56>",
69
- "<extra_id_17>",
70
- "<extra_id_9>",
71
- "<extra_id_81>",
72
- "<extra_id_20>",
73
- "<extra_id_30>",
74
  "<extra_id_44>",
75
- "<extra_id_77>",
76
- "<extra_id_89>",
 
 
 
 
 
77
  "<extra_id_52>",
78
- "<extra_id_83>",
79
- "<extra_id_85>",
80
- "<extra_id_75>",
81
- "<extra_id_0>",
82
- "<extra_id_12>",
83
  "<extra_id_54>",
84
- "<extra_id_62>",
85
- "<extra_id_76>",
86
- "<extra_id_70>",
87
- "<extra_id_28>",
88
- "<extra_id_8>",
89
- "<extra_id_84>",
90
- "<extra_id_2>",
91
  "<extra_id_61>",
92
- "<extra_id_10>",
93
- "<extra_id_53>",
94
- "<extra_id_6>",
 
 
95
  "<extra_id_67>",
 
 
 
96
  "<extra_id_71>",
97
- "<extra_id_66>",
98
- "<extra_id_46>",
99
- "<extra_id_18>",
100
- "<extra_id_38>",
101
- "<extra_id_59>",
102
- "<extra_id_40>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  ],
104
  "bos_token": {
105
  "content": "<s>",
@@ -116,7 +116,7 @@
116
  "single_word": false
117
  },
118
  "pad_token": {
119
- "content": "<pad>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
+ "<extra_id_0>",
 
 
 
 
 
 
 
4
  "<extra_id_1>",
5
+ "<extra_id_2>",
 
 
 
 
6
  "<extra_id_3>",
 
 
 
 
 
 
 
 
7
  "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
 
17
  "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
 
 
 
 
25
  "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
  "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
  "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
  "<extra_id_43>",
 
 
 
 
 
 
 
 
 
 
47
  "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
  "<extra_id_52>",
56
+ "<extra_id_53>",
 
 
 
 
57
  "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
 
64
  "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
  "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
  "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
  ],
104
  "bos_token": {
105
  "content": "<s>",
 
116
  "single_word": false
117
  },
118
  "pad_token": {
119
+ "content": "<s>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,
tokenizer.json CHANGED
@@ -41,7 +41,7 @@
41
  },
42
  {
43
  "id": 32000,
44
- "content": "<extra_id_39>",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
@@ -50,7 +50,7 @@
50
  },
51
  {
52
  "id": 32001,
53
- "content": "<extra_id_95>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
@@ -59,7 +59,7 @@
59
  },
60
  {
61
  "id": 32002,
62
- "content": "<extra_id_93>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
@@ -68,7 +68,7 @@
68
  },
69
  {
70
  "id": 32003,
71
- "content": "<extra_id_79>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": false,
@@ -77,7 +77,7 @@
77
  },
78
  {
79
  "id": 32004,
80
- "content": "<extra_id_32>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
@@ -86,7 +86,7 @@
86
  },
87
  {
88
  "id": 32005,
89
- "content": "<extra_id_15>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
@@ -95,7 +95,7 @@
95
  },
96
  {
97
  "id": 32006,
98
- "content": "<extra_id_73>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
@@ -104,7 +104,7 @@
104
  },
105
  {
106
  "id": 32007,
107
- "content": "<extra_id_36>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
@@ -113,7 +113,7 @@
113
  },
114
  {
115
  "id": 32008,
116
- "content": "<extra_id_1>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": false,
@@ -122,7 +122,7 @@
122
  },
123
  {
124
  "id": 32009,
125
- "content": "<extra_id_19>",
126
  "single_word": false,
127
  "lstrip": false,
128
  "rstrip": false,
@@ -131,7 +131,7 @@
131
  },
132
  {
133
  "id": 32010,
134
- "content": "<extra_id_26>",
135
  "single_word": false,
136
  "lstrip": false,
137
  "rstrip": false,
@@ -140,7 +140,7 @@
140
  },
141
  {
142
  "id": 32011,
143
- "content": "<extra_id_60>",
144
  "single_word": false,
145
  "lstrip": false,
146
  "rstrip": false,
@@ -149,7 +149,7 @@
149
  },
150
  {
151
  "id": 32012,
152
- "content": "<extra_id_51>",
153
  "single_word": false,
154
  "lstrip": false,
155
  "rstrip": false,
@@ -158,7 +158,7 @@
158
  },
159
  {
160
  "id": 32013,
161
- "content": "<extra_id_5>",
162
  "single_word": false,
163
  "lstrip": false,
164
  "rstrip": false,
@@ -167,7 +167,7 @@
167
  },
168
  {
169
  "id": 32014,
170
- "content": "<extra_id_3>",
171
  "single_word": false,
172
  "lstrip": false,
173
  "rstrip": false,
@@ -176,7 +176,7 @@
176
  },
177
  {
178
  "id": 32015,
179
- "content": "<extra_id_31>",
180
  "single_word": false,
181
  "lstrip": false,
182
  "rstrip": false,
@@ -185,7 +185,7 @@
185
  },
186
  {
187
  "id": 32016,
188
- "content": "<extra_id_58>",
189
  "single_word": false,
190
  "lstrip": false,
191
  "rstrip": false,
@@ -194,7 +194,7 @@
194
  },
195
  {
196
  "id": 32017,
197
- "content": "<extra_id_23>",
198
  "single_word": false,
199
  "lstrip": false,
200
  "rstrip": false,
@@ -203,7 +203,7 @@
203
  },
204
  {
205
  "id": 32018,
206
- "content": "<extra_id_50>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,
@@ -212,7 +212,7 @@
212
  },
213
  {
214
  "id": 32019,
215
- "content": "<extra_id_78>",
216
  "single_word": false,
217
  "lstrip": false,
218
  "rstrip": false,
@@ -221,7 +221,7 @@
221
  },
222
  {
223
  "id": 32020,
224
- "content": "<extra_id_96>",
225
  "single_word": false,
226
  "lstrip": false,
227
  "rstrip": false,
@@ -230,7 +230,7 @@
230
  },
231
  {
232
  "id": 32021,
233
- "content": "<extra_id_72>",
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
@@ -239,7 +239,7 @@
239
  },
240
  {
241
  "id": 32022,
242
- "content": "<extra_id_74>",
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
@@ -248,7 +248,7 @@
248
  },
249
  {
250
  "id": 32023,
251
- "content": "<extra_id_4>",
252
  "single_word": false,
253
  "lstrip": false,
254
  "rstrip": false,
@@ -257,7 +257,7 @@
257
  },
258
  {
259
  "id": 32024,
260
- "content": "<extra_id_48>",
261
  "single_word": false,
262
  "lstrip": false,
263
  "rstrip": false,
@@ -266,7 +266,7 @@
266
  },
267
  {
268
  "id": 32025,
269
- "content": "<extra_id_42>",
270
  "single_word": false,
271
  "lstrip": false,
272
  "rstrip": false,
@@ -275,7 +275,7 @@
275
  },
276
  {
277
  "id": 32026,
278
- "content": "<extra_id_90>",
279
  "single_word": false,
280
  "lstrip": false,
281
  "rstrip": false,
@@ -284,7 +284,7 @@
284
  },
285
  {
286
  "id": 32027,
287
- "content": "<extra_id_82>",
288
  "single_word": false,
289
  "lstrip": false,
290
  "rstrip": false,
@@ -293,7 +293,7 @@
293
  },
294
  {
295
  "id": 32028,
296
- "content": "<extra_id_57>",
297
  "single_word": false,
298
  "lstrip": false,
299
  "rstrip": false,
@@ -302,7 +302,7 @@
302
  },
303
  {
304
  "id": 32029,
305
- "content": "<extra_id_34>",
306
  "single_word": false,
307
  "lstrip": false,
308
  "rstrip": false,
@@ -311,7 +311,7 @@
311
  },
312
  {
313
  "id": 32030,
314
- "content": "<extra_id_16>",
315
  "single_word": false,
316
  "lstrip": false,
317
  "rstrip": false,
@@ -320,7 +320,7 @@
320
  },
321
  {
322
  "id": 32031,
323
- "content": "<extra_id_33>",
324
  "single_word": false,
325
  "lstrip": false,
326
  "rstrip": false,
@@ -329,7 +329,7 @@
329
  },
330
  {
331
  "id": 32032,
332
- "content": "<extra_id_35>",
333
  "single_word": false,
334
  "lstrip": false,
335
  "rstrip": false,
@@ -338,7 +338,7 @@
338
  },
339
  {
340
  "id": 32033,
341
- "content": "<extra_id_69>",
342
  "single_word": false,
343
  "lstrip": false,
344
  "rstrip": false,
@@ -347,7 +347,7 @@
347
  },
348
  {
349
  "id": 32034,
350
- "content": "<extra_id_14>",
351
  "single_word": false,
352
  "lstrip": false,
353
  "rstrip": false,
@@ -356,7 +356,7 @@
356
  },
357
  {
358
  "id": 32035,
359
- "content": "<extra_id_65>",
360
  "single_word": false,
361
  "lstrip": false,
362
  "rstrip": false,
@@ -365,7 +365,7 @@
365
  },
366
  {
367
  "id": 32036,
368
- "content": "<extra_id_88>",
369
  "single_word": false,
370
  "lstrip": false,
371
  "rstrip": false,
@@ -374,7 +374,7 @@
374
  },
375
  {
376
  "id": 32037,
377
- "content": "<extra_id_97>",
378
  "single_word": false,
379
  "lstrip": false,
380
  "rstrip": false,
@@ -383,7 +383,7 @@
383
  },
384
  {
385
  "id": 32038,
386
- "content": "<extra_id_94>",
387
  "single_word": false,
388
  "lstrip": false,
389
  "rstrip": false,
@@ -392,7 +392,7 @@
392
  },
393
  {
394
  "id": 32039,
395
- "content": "<extra_id_27>",
396
  "single_word": false,
397
  "lstrip": false,
398
  "rstrip": false,
@@ -401,7 +401,7 @@
401
  },
402
  {
403
  "id": 32040,
404
- "content": "<extra_id_80>",
405
  "single_word": false,
406
  "lstrip": false,
407
  "rstrip": false,
@@ -419,7 +419,7 @@
419
  },
420
  {
421
  "id": 32042,
422
- "content": "<extra_id_55>",
423
  "single_word": false,
424
  "lstrip": false,
425
  "rstrip": false,
@@ -428,7 +428,7 @@
428
  },
429
  {
430
  "id": 32043,
431
- "content": "<extra_id_68>",
432
  "single_word": false,
433
  "lstrip": false,
434
  "rstrip": false,
@@ -437,7 +437,7 @@
437
  },
438
  {
439
  "id": 32044,
440
- "content": "<extra_id_98>",
441
  "single_word": false,
442
  "lstrip": false,
443
  "rstrip": false,
@@ -446,7 +446,7 @@
446
  },
447
  {
448
  "id": 32045,
449
- "content": "<extra_id_24>",
450
  "single_word": false,
451
  "lstrip": false,
452
  "rstrip": false,
@@ -455,7 +455,7 @@
455
  },
456
  {
457
  "id": 32046,
458
- "content": "<extra_id_22>",
459
  "single_word": false,
460
  "lstrip": false,
461
  "rstrip": false,
@@ -464,7 +464,7 @@
464
  },
465
  {
466
  "id": 32047,
467
- "content": "<extra_id_49>",
468
  "single_word": false,
469
  "lstrip": false,
470
  "rstrip": false,
@@ -473,7 +473,7 @@
473
  },
474
  {
475
  "id": 32048,
476
- "content": "<extra_id_91>",
477
  "single_word": false,
478
  "lstrip": false,
479
  "rstrip": false,
@@ -482,7 +482,7 @@
482
  },
483
  {
484
  "id": 32049,
485
- "content": "<extra_id_64>",
486
  "single_word": false,
487
  "lstrip": false,
488
  "rstrip": false,
@@ -491,7 +491,7 @@
491
  },
492
  {
493
  "id": 32050,
494
- "content": "<extra_id_99>",
495
  "single_word": false,
496
  "lstrip": false,
497
  "rstrip": false,
@@ -500,7 +500,7 @@
500
  },
501
  {
502
  "id": 32051,
503
- "content": "<extra_id_13>",
504
  "single_word": false,
505
  "lstrip": false,
506
  "rstrip": false,
@@ -509,7 +509,7 @@
509
  },
510
  {
511
  "id": 32052,
512
- "content": "<extra_id_45>",
513
  "single_word": false,
514
  "lstrip": false,
515
  "rstrip": false,
@@ -518,7 +518,7 @@
518
  },
519
  {
520
  "id": 32053,
521
- "content": "<extra_id_29>",
522
  "single_word": false,
523
  "lstrip": false,
524
  "rstrip": false,
@@ -527,7 +527,7 @@
527
  },
528
  {
529
  "id": 32054,
530
- "content": "<extra_id_11>",
531
  "single_word": false,
532
  "lstrip": false,
533
  "rstrip": false,
@@ -536,7 +536,7 @@
536
  },
537
  {
538
  "id": 32055,
539
- "content": "<extra_id_87>",
540
  "single_word": false,
541
  "lstrip": false,
542
  "rstrip": false,
@@ -545,7 +545,7 @@
545
  },
546
  {
547
  "id": 32056,
548
- "content": "<extra_id_47>",
549
  "single_word": false,
550
  "lstrip": false,
551
  "rstrip": false,
@@ -554,7 +554,7 @@
554
  },
555
  {
556
  "id": 32057,
557
- "content": "<extra_id_37>",
558
  "single_word": false,
559
  "lstrip": false,
560
  "rstrip": false,
@@ -563,7 +563,7 @@
563
  },
564
  {
565
  "id": 32058,
566
- "content": "<extra_id_86>",
567
  "single_word": false,
568
  "lstrip": false,
569
  "rstrip": false,
@@ -572,7 +572,7 @@
572
  },
573
  {
574
  "id": 32059,
575
- "content": "<extra_id_63>",
576
  "single_word": false,
577
  "lstrip": false,
578
  "rstrip": false,
@@ -581,7 +581,7 @@
581
  },
582
  {
583
  "id": 32060,
584
- "content": "<extra_id_43>",
585
  "single_word": false,
586
  "lstrip": false,
587
  "rstrip": false,
@@ -590,7 +590,7 @@
590
  },
591
  {
592
  "id": 32061,
593
- "content": "<extra_id_25>",
594
  "single_word": false,
595
  "lstrip": false,
596
  "rstrip": false,
@@ -599,7 +599,7 @@
599
  },
600
  {
601
  "id": 32062,
602
- "content": "<extra_id_21>",
603
  "single_word": false,
604
  "lstrip": false,
605
  "rstrip": false,
@@ -608,7 +608,7 @@
608
  },
609
  {
610
  "id": 32063,
611
- "content": "<extra_id_7>",
612
  "single_word": false,
613
  "lstrip": false,
614
  "rstrip": false,
@@ -617,7 +617,7 @@
617
  },
618
  {
619
  "id": 32064,
620
- "content": "<extra_id_92>",
621
  "single_word": false,
622
  "lstrip": false,
623
  "rstrip": false,
@@ -626,7 +626,7 @@
626
  },
627
  {
628
  "id": 32065,
629
- "content": "<extra_id_56>",
630
  "single_word": false,
631
  "lstrip": false,
632
  "rstrip": false,
@@ -635,7 +635,7 @@
635
  },
636
  {
637
  "id": 32066,
638
- "content": "<extra_id_17>",
639
  "single_word": false,
640
  "lstrip": false,
641
  "rstrip": false,
@@ -644,7 +644,7 @@
644
  },
645
  {
646
  "id": 32067,
647
- "content": "<extra_id_9>",
648
  "single_word": false,
649
  "lstrip": false,
650
  "rstrip": false,
@@ -653,7 +653,7 @@
653
  },
654
  {
655
  "id": 32068,
656
- "content": "<extra_id_81>",
657
  "single_word": false,
658
  "lstrip": false,
659
  "rstrip": false,
@@ -662,7 +662,7 @@
662
  },
663
  {
664
  "id": 32069,
665
- "content": "<extra_id_20>",
666
  "single_word": false,
667
  "lstrip": false,
668
  "rstrip": false,
@@ -671,7 +671,7 @@
671
  },
672
  {
673
  "id": 32070,
674
- "content": "<extra_id_30>",
675
  "single_word": false,
676
  "lstrip": false,
677
  "rstrip": false,
@@ -680,7 +680,7 @@
680
  },
681
  {
682
  "id": 32071,
683
- "content": "<extra_id_44>",
684
  "single_word": false,
685
  "lstrip": false,
686
  "rstrip": false,
@@ -689,7 +689,7 @@
689
  },
690
  {
691
  "id": 32072,
692
- "content": "<extra_id_77>",
693
  "single_word": false,
694
  "lstrip": false,
695
  "rstrip": false,
@@ -698,7 +698,7 @@
698
  },
699
  {
700
  "id": 32073,
701
- "content": "<extra_id_89>",
702
  "single_word": false,
703
  "lstrip": false,
704
  "rstrip": false,
@@ -707,7 +707,7 @@
707
  },
708
  {
709
  "id": 32074,
710
- "content": "<extra_id_52>",
711
  "single_word": false,
712
  "lstrip": false,
713
  "rstrip": false,
@@ -716,7 +716,7 @@
716
  },
717
  {
718
  "id": 32075,
719
- "content": "<extra_id_83>",
720
  "single_word": false,
721
  "lstrip": false,
722
  "rstrip": false,
@@ -725,7 +725,7 @@
725
  },
726
  {
727
  "id": 32076,
728
- "content": "<extra_id_85>",
729
  "single_word": false,
730
  "lstrip": false,
731
  "rstrip": false,
@@ -734,7 +734,7 @@
734
  },
735
  {
736
  "id": 32077,
737
- "content": "<extra_id_75>",
738
  "single_word": false,
739
  "lstrip": false,
740
  "rstrip": false,
@@ -743,7 +743,7 @@
743
  },
744
  {
745
  "id": 32078,
746
- "content": "<extra_id_0>",
747
  "single_word": false,
748
  "lstrip": false,
749
  "rstrip": false,
@@ -752,7 +752,7 @@
752
  },
753
  {
754
  "id": 32079,
755
- "content": "<extra_id_12>",
756
  "single_word": false,
757
  "lstrip": false,
758
  "rstrip": false,
@@ -761,7 +761,7 @@
761
  },
762
  {
763
  "id": 32080,
764
- "content": "<extra_id_54>",
765
  "single_word": false,
766
  "lstrip": false,
767
  "rstrip": false,
@@ -770,7 +770,7 @@
770
  },
771
  {
772
  "id": 32081,
773
- "content": "<extra_id_62>",
774
  "single_word": false,
775
  "lstrip": false,
776
  "rstrip": false,
@@ -779,7 +779,7 @@
779
  },
780
  {
781
  "id": 32082,
782
- "content": "<extra_id_76>",
783
  "single_word": false,
784
  "lstrip": false,
785
  "rstrip": false,
@@ -788,7 +788,7 @@
788
  },
789
  {
790
  "id": 32083,
791
- "content": "<extra_id_70>",
792
  "single_word": false,
793
  "lstrip": false,
794
  "rstrip": false,
@@ -797,7 +797,7 @@
797
  },
798
  {
799
  "id": 32084,
800
- "content": "<extra_id_28>",
801
  "single_word": false,
802
  "lstrip": false,
803
  "rstrip": false,
@@ -806,7 +806,7 @@
806
  },
807
  {
808
  "id": 32085,
809
- "content": "<extra_id_8>",
810
  "single_word": false,
811
  "lstrip": false,
812
  "rstrip": false,
@@ -815,7 +815,7 @@
815
  },
816
  {
817
  "id": 32086,
818
- "content": "<extra_id_84>",
819
  "single_word": false,
820
  "lstrip": false,
821
  "rstrip": false,
@@ -824,7 +824,7 @@
824
  },
825
  {
826
  "id": 32087,
827
- "content": "<extra_id_2>",
828
  "single_word": false,
829
  "lstrip": false,
830
  "rstrip": false,
@@ -833,7 +833,7 @@
833
  },
834
  {
835
  "id": 32088,
836
- "content": "<extra_id_61>",
837
  "single_word": false,
838
  "lstrip": false,
839
  "rstrip": false,
@@ -842,7 +842,7 @@
842
  },
843
  {
844
  "id": 32089,
845
- "content": "<extra_id_10>",
846
  "single_word": false,
847
  "lstrip": false,
848
  "rstrip": false,
@@ -851,7 +851,7 @@
851
  },
852
  {
853
  "id": 32090,
854
- "content": "<extra_id_53>",
855
  "single_word": false,
856
  "lstrip": false,
857
  "rstrip": false,
@@ -860,7 +860,7 @@
860
  },
861
  {
862
  "id": 32091,
863
- "content": "<extra_id_6>",
864
  "single_word": false,
865
  "lstrip": false,
866
  "rstrip": false,
@@ -869,7 +869,7 @@
869
  },
870
  {
871
  "id": 32092,
872
- "content": "<extra_id_67>",
873
  "single_word": false,
874
  "lstrip": false,
875
  "rstrip": false,
@@ -878,7 +878,7 @@
878
  },
879
  {
880
  "id": 32093,
881
- "content": "<extra_id_71>",
882
  "single_word": false,
883
  "lstrip": false,
884
  "rstrip": false,
@@ -887,7 +887,7 @@
887
  },
888
  {
889
  "id": 32094,
890
- "content": "<extra_id_66>",
891
  "single_word": false,
892
  "lstrip": false,
893
  "rstrip": false,
@@ -896,7 +896,7 @@
896
  },
897
  {
898
  "id": 32095,
899
- "content": "<extra_id_46>",
900
  "single_word": false,
901
  "lstrip": false,
902
  "rstrip": false,
@@ -905,7 +905,7 @@
905
  },
906
  {
907
  "id": 32096,
908
- "content": "<extra_id_18>",
909
  "single_word": false,
910
  "lstrip": false,
911
  "rstrip": false,
@@ -914,7 +914,7 @@
914
  },
915
  {
916
  "id": 32097,
917
- "content": "<extra_id_38>",
918
  "single_word": false,
919
  "lstrip": false,
920
  "rstrip": false,
@@ -923,7 +923,7 @@
923
  },
924
  {
925
  "id": 32098,
926
- "content": "<extra_id_59>",
927
  "single_word": false,
928
  "lstrip": false,
929
  "rstrip": false,
@@ -932,7 +932,7 @@
932
  },
933
  {
934
  "id": 32099,
935
- "content": "<extra_id_40>",
936
  "single_word": false,
937
  "lstrip": false,
938
  "rstrip": false,
 
41
  },
42
  {
43
  "id": 32000,
44
+ "content": "<extra_id_0>",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
 
50
  },
51
  {
52
  "id": 32001,
53
+ "content": "<extra_id_1>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
 
59
  },
60
  {
61
  "id": 32002,
62
+ "content": "<extra_id_2>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
 
68
  },
69
  {
70
  "id": 32003,
71
+ "content": "<extra_id_3>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": false,
 
77
  },
78
  {
79
  "id": 32004,
80
+ "content": "<extra_id_4>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
 
86
  },
87
  {
88
  "id": 32005,
89
+ "content": "<extra_id_5>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
 
95
  },
96
  {
97
  "id": 32006,
98
+ "content": "<extra_id_6>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
 
104
  },
105
  {
106
  "id": 32007,
107
+ "content": "<extra_id_7>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
 
113
  },
114
  {
115
  "id": 32008,
116
+ "content": "<extra_id_8>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": false,
 
122
  },
123
  {
124
  "id": 32009,
125
+ "content": "<extra_id_9>",
126
  "single_word": false,
127
  "lstrip": false,
128
  "rstrip": false,
 
131
  },
132
  {
133
  "id": 32010,
134
+ "content": "<extra_id_10>",
135
  "single_word": false,
136
  "lstrip": false,
137
  "rstrip": false,
 
140
  },
141
  {
142
  "id": 32011,
143
+ "content": "<extra_id_11>",
144
  "single_word": false,
145
  "lstrip": false,
146
  "rstrip": false,
 
149
  },
150
  {
151
  "id": 32012,
152
+ "content": "<extra_id_12>",
153
  "single_word": false,
154
  "lstrip": false,
155
  "rstrip": false,
 
158
  },
159
  {
160
  "id": 32013,
161
+ "content": "<extra_id_13>",
162
  "single_word": false,
163
  "lstrip": false,
164
  "rstrip": false,
 
167
  },
168
  {
169
  "id": 32014,
170
+ "content": "<extra_id_14>",
171
  "single_word": false,
172
  "lstrip": false,
173
  "rstrip": false,
 
176
  },
177
  {
178
  "id": 32015,
179
+ "content": "<extra_id_15>",
180
  "single_word": false,
181
  "lstrip": false,
182
  "rstrip": false,
 
185
  },
186
  {
187
  "id": 32016,
188
+ "content": "<extra_id_16>",
189
  "single_word": false,
190
  "lstrip": false,
191
  "rstrip": false,
 
194
  },
195
  {
196
  "id": 32017,
197
+ "content": "<extra_id_17>",
198
  "single_word": false,
199
  "lstrip": false,
200
  "rstrip": false,
 
203
  },
204
  {
205
  "id": 32018,
206
+ "content": "<extra_id_18>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,
 
212
  },
213
  {
214
  "id": 32019,
215
+ "content": "<extra_id_19>",
216
  "single_word": false,
217
  "lstrip": false,
218
  "rstrip": false,
 
221
  },
222
  {
223
  "id": 32020,
224
+ "content": "<extra_id_20>",
225
  "single_word": false,
226
  "lstrip": false,
227
  "rstrip": false,
 
230
  },
231
  {
232
  "id": 32021,
233
+ "content": "<extra_id_21>",
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
 
239
  },
240
  {
241
  "id": 32022,
242
+ "content": "<extra_id_22>",
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
 
248
  },
249
  {
250
  "id": 32023,
251
+ "content": "<extra_id_23>",
252
  "single_word": false,
253
  "lstrip": false,
254
  "rstrip": false,
 
257
  },
258
  {
259
  "id": 32024,
260
+ "content": "<extra_id_24>",
261
  "single_word": false,
262
  "lstrip": false,
263
  "rstrip": false,
 
266
  },
267
  {
268
  "id": 32025,
269
+ "content": "<extra_id_25>",
270
  "single_word": false,
271
  "lstrip": false,
272
  "rstrip": false,
 
275
  },
276
  {
277
  "id": 32026,
278
+ "content": "<extra_id_26>",
279
  "single_word": false,
280
  "lstrip": false,
281
  "rstrip": false,
 
284
  },
285
  {
286
  "id": 32027,
287
+ "content": "<extra_id_27>",
288
  "single_word": false,
289
  "lstrip": false,
290
  "rstrip": false,
 
293
  },
294
  {
295
  "id": 32028,
296
+ "content": "<extra_id_28>",
297
  "single_word": false,
298
  "lstrip": false,
299
  "rstrip": false,
 
302
  },
303
  {
304
  "id": 32029,
305
+ "content": "<extra_id_29>",
306
  "single_word": false,
307
  "lstrip": false,
308
  "rstrip": false,
 
311
  },
312
  {
313
  "id": 32030,
314
+ "content": "<extra_id_30>",
315
  "single_word": false,
316
  "lstrip": false,
317
  "rstrip": false,
 
320
  },
321
  {
322
  "id": 32031,
323
+ "content": "<extra_id_31>",
324
  "single_word": false,
325
  "lstrip": false,
326
  "rstrip": false,
 
329
  },
330
  {
331
  "id": 32032,
332
+ "content": "<extra_id_32>",
333
  "single_word": false,
334
  "lstrip": false,
335
  "rstrip": false,
 
338
  },
339
  {
340
  "id": 32033,
341
+ "content": "<extra_id_33>",
342
  "single_word": false,
343
  "lstrip": false,
344
  "rstrip": false,
 
347
  },
348
  {
349
  "id": 32034,
350
+ "content": "<extra_id_34>",
351
  "single_word": false,
352
  "lstrip": false,
353
  "rstrip": false,
 
356
  },
357
  {
358
  "id": 32035,
359
+ "content": "<extra_id_35>",
360
  "single_word": false,
361
  "lstrip": false,
362
  "rstrip": false,
 
365
  },
366
  {
367
  "id": 32036,
368
+ "content": "<extra_id_36>",
369
  "single_word": false,
370
  "lstrip": false,
371
  "rstrip": false,
 
374
  },
375
  {
376
  "id": 32037,
377
+ "content": "<extra_id_37>",
378
  "single_word": false,
379
  "lstrip": false,
380
  "rstrip": false,
 
383
  },
384
  {
385
  "id": 32038,
386
+ "content": "<extra_id_38>",
387
  "single_word": false,
388
  "lstrip": false,
389
  "rstrip": false,
 
392
  },
393
  {
394
  "id": 32039,
395
+ "content": "<extra_id_39>",
396
  "single_word": false,
397
  "lstrip": false,
398
  "rstrip": false,
 
401
  },
402
  {
403
  "id": 32040,
404
+ "content": "<extra_id_40>",
405
  "single_word": false,
406
  "lstrip": false,
407
  "rstrip": false,
 
419
  },
420
  {
421
  "id": 32042,
422
+ "content": "<extra_id_42>",
423
  "single_word": false,
424
  "lstrip": false,
425
  "rstrip": false,
 
428
  },
429
  {
430
  "id": 32043,
431
+ "content": "<extra_id_43>",
432
  "single_word": false,
433
  "lstrip": false,
434
  "rstrip": false,
 
437
  },
438
  {
439
  "id": 32044,
440
+ "content": "<extra_id_44>",
441
  "single_word": false,
442
  "lstrip": false,
443
  "rstrip": false,
 
446
  },
447
  {
448
  "id": 32045,
449
+ "content": "<extra_id_45>",
450
  "single_word": false,
451
  "lstrip": false,
452
  "rstrip": false,
 
455
  },
456
  {
457
  "id": 32046,
458
+ "content": "<extra_id_46>",
459
  "single_word": false,
460
  "lstrip": false,
461
  "rstrip": false,
 
464
  },
465
  {
466
  "id": 32047,
467
+ "content": "<extra_id_47>",
468
  "single_word": false,
469
  "lstrip": false,
470
  "rstrip": false,
 
473
  },
474
  {
475
  "id": 32048,
476
+ "content": "<extra_id_48>",
477
  "single_word": false,
478
  "lstrip": false,
479
  "rstrip": false,
 
482
  },
483
  {
484
  "id": 32049,
485
+ "content": "<extra_id_49>",
486
  "single_word": false,
487
  "lstrip": false,
488
  "rstrip": false,
 
491
  },
492
  {
493
  "id": 32050,
494
+ "content": "<extra_id_50>",
495
  "single_word": false,
496
  "lstrip": false,
497
  "rstrip": false,
 
500
  },
501
  {
502
  "id": 32051,
503
+ "content": "<extra_id_51>",
504
  "single_word": false,
505
  "lstrip": false,
506
  "rstrip": false,
 
509
  },
510
  {
511
  "id": 32052,
512
+ "content": "<extra_id_52>",
513
  "single_word": false,
514
  "lstrip": false,
515
  "rstrip": false,
 
518
  },
519
  {
520
  "id": 32053,
521
+ "content": "<extra_id_53>",
522
  "single_word": false,
523
  "lstrip": false,
524
  "rstrip": false,
 
527
  },
528
  {
529
  "id": 32054,
530
+ "content": "<extra_id_54>",
531
  "single_word": false,
532
  "lstrip": false,
533
  "rstrip": false,
 
536
  },
537
  {
538
  "id": 32055,
539
+ "content": "<extra_id_55>",
540
  "single_word": false,
541
  "lstrip": false,
542
  "rstrip": false,
 
545
  },
546
  {
547
  "id": 32056,
548
+ "content": "<extra_id_56>",
549
  "single_word": false,
550
  "lstrip": false,
551
  "rstrip": false,
 
554
  },
555
  {
556
  "id": 32057,
557
+ "content": "<extra_id_57>",
558
  "single_word": false,
559
  "lstrip": false,
560
  "rstrip": false,
 
563
  },
564
  {
565
  "id": 32058,
566
+ "content": "<extra_id_58>",
567
  "single_word": false,
568
  "lstrip": false,
569
  "rstrip": false,
 
572
  },
573
  {
574
  "id": 32059,
575
+ "content": "<extra_id_59>",
576
  "single_word": false,
577
  "lstrip": false,
578
  "rstrip": false,
 
581
  },
582
  {
583
  "id": 32060,
584
+ "content": "<extra_id_60>",
585
  "single_word": false,
586
  "lstrip": false,
587
  "rstrip": false,
 
590
  },
591
  {
592
  "id": 32061,
593
+ "content": "<extra_id_61>",
594
  "single_word": false,
595
  "lstrip": false,
596
  "rstrip": false,
 
599
  },
600
  {
601
  "id": 32062,
602
+ "content": "<extra_id_62>",
603
  "single_word": false,
604
  "lstrip": false,
605
  "rstrip": false,
 
608
  },
609
  {
610
  "id": 32063,
611
+ "content": "<extra_id_63>",
612
  "single_word": false,
613
  "lstrip": false,
614
  "rstrip": false,
 
617
  },
618
  {
619
  "id": 32064,
620
+ "content": "<extra_id_64>",
621
  "single_word": false,
622
  "lstrip": false,
623
  "rstrip": false,
 
626
  },
627
  {
628
  "id": 32065,
629
+ "content": "<extra_id_65>",
630
  "single_word": false,
631
  "lstrip": false,
632
  "rstrip": false,
 
635
  },
636
  {
637
  "id": 32066,
638
+ "content": "<extra_id_66>",
639
  "single_word": false,
640
  "lstrip": false,
641
  "rstrip": false,
 
644
  },
645
  {
646
  "id": 32067,
647
+ "content": "<extra_id_67>",
648
  "single_word": false,
649
  "lstrip": false,
650
  "rstrip": false,
 
653
  },
654
  {
655
  "id": 32068,
656
+ "content": "<extra_id_68>",
657
  "single_word": false,
658
  "lstrip": false,
659
  "rstrip": false,
 
662
  },
663
  {
664
  "id": 32069,
665
+ "content": "<extra_id_69>",
666
  "single_word": false,
667
  "lstrip": false,
668
  "rstrip": false,
 
671
  },
672
  {
673
  "id": 32070,
674
+ "content": "<extra_id_70>",
675
  "single_word": false,
676
  "lstrip": false,
677
  "rstrip": false,
 
680
  },
681
  {
682
  "id": 32071,
683
+ "content": "<extra_id_71>",
684
  "single_word": false,
685
  "lstrip": false,
686
  "rstrip": false,
 
689
  },
690
  {
691
  "id": 32072,
692
+ "content": "<extra_id_72>",
693
  "single_word": false,
694
  "lstrip": false,
695
  "rstrip": false,
 
698
  },
699
  {
700
  "id": 32073,
701
+ "content": "<extra_id_73>",
702
  "single_word": false,
703
  "lstrip": false,
704
  "rstrip": false,
 
707
  },
708
  {
709
  "id": 32074,
710
+ "content": "<extra_id_74>",
711
  "single_word": false,
712
  "lstrip": false,
713
  "rstrip": false,
 
716
  },
717
  {
718
  "id": 32075,
719
+ "content": "<extra_id_75>",
720
  "single_word": false,
721
  "lstrip": false,
722
  "rstrip": false,
 
725
  },
726
  {
727
  "id": 32076,
728
+ "content": "<extra_id_76>",
729
  "single_word": false,
730
  "lstrip": false,
731
  "rstrip": false,
 
734
  },
735
  {
736
  "id": 32077,
737
+ "content": "<extra_id_77>",
738
  "single_word": false,
739
  "lstrip": false,
740
  "rstrip": false,
 
743
  },
744
  {
745
  "id": 32078,
746
+ "content": "<extra_id_78>",
747
  "single_word": false,
748
  "lstrip": false,
749
  "rstrip": false,
 
752
  },
753
  {
754
  "id": 32079,
755
+ "content": "<extra_id_79>",
756
  "single_word": false,
757
  "lstrip": false,
758
  "rstrip": false,
 
761
  },
762
  {
763
  "id": 32080,
764
+ "content": "<extra_id_80>",
765
  "single_word": false,
766
  "lstrip": false,
767
  "rstrip": false,
 
770
  },
771
  {
772
  "id": 32081,
773
+ "content": "<extra_id_81>",
774
  "single_word": false,
775
  "lstrip": false,
776
  "rstrip": false,
 
779
  },
780
  {
781
  "id": 32082,
782
+ "content": "<extra_id_82>",
783
  "single_word": false,
784
  "lstrip": false,
785
  "rstrip": false,
 
788
  },
789
  {
790
  "id": 32083,
791
+ "content": "<extra_id_83>",
792
  "single_word": false,
793
  "lstrip": false,
794
  "rstrip": false,
 
797
  },
798
  {
799
  "id": 32084,
800
+ "content": "<extra_id_84>",
801
  "single_word": false,
802
  "lstrip": false,
803
  "rstrip": false,
 
806
  },
807
  {
808
  "id": 32085,
809
+ "content": "<extra_id_85>",
810
  "single_word": false,
811
  "lstrip": false,
812
  "rstrip": false,
 
815
  },
816
  {
817
  "id": 32086,
818
+ "content": "<extra_id_86>",
819
  "single_word": false,
820
  "lstrip": false,
821
  "rstrip": false,
 
824
  },
825
  {
826
  "id": 32087,
827
+ "content": "<extra_id_87>",
828
  "single_word": false,
829
  "lstrip": false,
830
  "rstrip": false,
 
833
  },
834
  {
835
  "id": 32088,
836
+ "content": "<extra_id_88>",
837
  "single_word": false,
838
  "lstrip": false,
839
  "rstrip": false,
 
842
  },
843
  {
844
  "id": 32089,
845
+ "content": "<extra_id_89>",
846
  "single_word": false,
847
  "lstrip": false,
848
  "rstrip": false,
 
851
  },
852
  {
853
  "id": 32090,
854
+ "content": "<extra_id_90>",
855
  "single_word": false,
856
  "lstrip": false,
857
  "rstrip": false,
 
860
  },
861
  {
862
  "id": 32091,
863
+ "content": "<extra_id_91>",
864
  "single_word": false,
865
  "lstrip": false,
866
  "rstrip": false,
 
869
  },
870
  {
871
  "id": 32092,
872
+ "content": "<extra_id_92>",
873
  "single_word": false,
874
  "lstrip": false,
875
  "rstrip": false,
 
878
  },
879
  {
880
  "id": 32093,
881
+ "content": "<extra_id_93>",
882
  "single_word": false,
883
  "lstrip": false,
884
  "rstrip": false,
 
887
  },
888
  {
889
  "id": 32094,
890
+ "content": "<extra_id_94>",
891
  "single_word": false,
892
  "lstrip": false,
893
  "rstrip": false,
 
896
  },
897
  {
898
  "id": 32095,
899
+ "content": "<extra_id_95>",
900
  "single_word": false,
901
  "lstrip": false,
902
  "rstrip": false,
 
905
  },
906
  {
907
  "id": 32096,
908
+ "content": "<extra_id_96>",
909
  "single_word": false,
910
  "lstrip": false,
911
  "rstrip": false,
 
914
  },
915
  {
916
  "id": 32097,
917
+ "content": "<extra_id_97>",
918
  "single_word": false,
919
  "lstrip": false,
920
  "rstrip": false,
 
923
  },
924
  {
925
  "id": 32098,
926
+ "content": "<extra_id_98>",
927
  "single_word": false,
928
  "lstrip": false,
929
  "rstrip": false,
 
932
  },
933
  {
934
  "id": 32099,
935
+ "content": "<extra_id_99>",
936
  "single_word": false,
937
  "lstrip": false,
938
  "rstrip": false,
tokenizer_config.json CHANGED
@@ -33,7 +33,7 @@
33
  "special": true
34
  },
35
  "32000": {
36
- "content": "<extra_id_39>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
@@ -41,7 +41,7 @@
41
  "special": true
42
  },
43
  "32001": {
44
- "content": "<extra_id_95>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
@@ -49,7 +49,7 @@
49
  "special": true
50
  },
51
  "32002": {
52
- "content": "<extra_id_93>",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
@@ -57,7 +57,7 @@
57
  "special": true
58
  },
59
  "32003": {
60
- "content": "<extra_id_79>",
61
  "lstrip": false,
62
  "normalized": false,
63
  "rstrip": false,
@@ -65,7 +65,7 @@
65
  "special": true
66
  },
67
  "32004": {
68
- "content": "<extra_id_32>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
@@ -73,7 +73,7 @@
73
  "special": true
74
  },
75
  "32005": {
76
- "content": "<extra_id_15>",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
@@ -81,7 +81,7 @@
81
  "special": true
82
  },
83
  "32006": {
84
- "content": "<extra_id_73>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
@@ -89,7 +89,7 @@
89
  "special": true
90
  },
91
  "32007": {
92
- "content": "<extra_id_36>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
@@ -97,7 +97,7 @@
97
  "special": true
98
  },
99
  "32008": {
100
- "content": "<extra_id_1>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
@@ -105,7 +105,7 @@
105
  "special": true
106
  },
107
  "32009": {
108
- "content": "<extra_id_19>",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
@@ -113,7 +113,7 @@
113
  "special": true
114
  },
115
  "32010": {
116
- "content": "<extra_id_26>",
117
  "lstrip": false,
118
  "normalized": false,
119
  "rstrip": false,
@@ -121,7 +121,7 @@
121
  "special": true
122
  },
123
  "32011": {
124
- "content": "<extra_id_60>",
125
  "lstrip": false,
126
  "normalized": false,
127
  "rstrip": false,
@@ -129,7 +129,7 @@
129
  "special": true
130
  },
131
  "32012": {
132
- "content": "<extra_id_51>",
133
  "lstrip": false,
134
  "normalized": false,
135
  "rstrip": false,
@@ -137,7 +137,7 @@
137
  "special": true
138
  },
139
  "32013": {
140
- "content": "<extra_id_5>",
141
  "lstrip": false,
142
  "normalized": false,
143
  "rstrip": false,
@@ -145,7 +145,7 @@
145
  "special": true
146
  },
147
  "32014": {
148
- "content": "<extra_id_3>",
149
  "lstrip": false,
150
  "normalized": false,
151
  "rstrip": false,
@@ -153,7 +153,7 @@
153
  "special": true
154
  },
155
  "32015": {
156
- "content": "<extra_id_31>",
157
  "lstrip": false,
158
  "normalized": false,
159
  "rstrip": false,
@@ -161,7 +161,7 @@
161
  "special": true
162
  },
163
  "32016": {
164
- "content": "<extra_id_58>",
165
  "lstrip": false,
166
  "normalized": false,
167
  "rstrip": false,
@@ -169,7 +169,7 @@
169
  "special": true
170
  },
171
  "32017": {
172
- "content": "<extra_id_23>",
173
  "lstrip": false,
174
  "normalized": false,
175
  "rstrip": false,
@@ -177,7 +177,7 @@
177
  "special": true
178
  },
179
  "32018": {
180
- "content": "<extra_id_50>",
181
  "lstrip": false,
182
  "normalized": false,
183
  "rstrip": false,
@@ -185,7 +185,7 @@
185
  "special": true
186
  },
187
  "32019": {
188
- "content": "<extra_id_78>",
189
  "lstrip": false,
190
  "normalized": false,
191
  "rstrip": false,
@@ -193,7 +193,7 @@
193
  "special": true
194
  },
195
  "32020": {
196
- "content": "<extra_id_96>",
197
  "lstrip": false,
198
  "normalized": false,
199
  "rstrip": false,
@@ -201,7 +201,7 @@
201
  "special": true
202
  },
203
  "32021": {
204
- "content": "<extra_id_72>",
205
  "lstrip": false,
206
  "normalized": false,
207
  "rstrip": false,
@@ -209,7 +209,7 @@
209
  "special": true
210
  },
211
  "32022": {
212
- "content": "<extra_id_74>",
213
  "lstrip": false,
214
  "normalized": false,
215
  "rstrip": false,
@@ -217,7 +217,7 @@
217
  "special": true
218
  },
219
  "32023": {
220
- "content": "<extra_id_4>",
221
  "lstrip": false,
222
  "normalized": false,
223
  "rstrip": false,
@@ -225,7 +225,7 @@
225
  "special": true
226
  },
227
  "32024": {
228
- "content": "<extra_id_48>",
229
  "lstrip": false,
230
  "normalized": false,
231
  "rstrip": false,
@@ -233,7 +233,7 @@
233
  "special": true
234
  },
235
  "32025": {
236
- "content": "<extra_id_42>",
237
  "lstrip": false,
238
  "normalized": false,
239
  "rstrip": false,
@@ -241,7 +241,7 @@
241
  "special": true
242
  },
243
  "32026": {
244
- "content": "<extra_id_90>",
245
  "lstrip": false,
246
  "normalized": false,
247
  "rstrip": false,
@@ -249,7 +249,7 @@
249
  "special": true
250
  },
251
  "32027": {
252
- "content": "<extra_id_82>",
253
  "lstrip": false,
254
  "normalized": false,
255
  "rstrip": false,
@@ -257,7 +257,7 @@
257
  "special": true
258
  },
259
  "32028": {
260
- "content": "<extra_id_57>",
261
  "lstrip": false,
262
  "normalized": false,
263
  "rstrip": false,
@@ -265,7 +265,7 @@
265
  "special": true
266
  },
267
  "32029": {
268
- "content": "<extra_id_34>",
269
  "lstrip": false,
270
  "normalized": false,
271
  "rstrip": false,
@@ -273,7 +273,7 @@
273
  "special": true
274
  },
275
  "32030": {
276
- "content": "<extra_id_16>",
277
  "lstrip": false,
278
  "normalized": false,
279
  "rstrip": false,
@@ -281,7 +281,7 @@
281
  "special": true
282
  },
283
  "32031": {
284
- "content": "<extra_id_33>",
285
  "lstrip": false,
286
  "normalized": false,
287
  "rstrip": false,
@@ -289,7 +289,7 @@
289
  "special": true
290
  },
291
  "32032": {
292
- "content": "<extra_id_35>",
293
  "lstrip": false,
294
  "normalized": false,
295
  "rstrip": false,
@@ -297,7 +297,7 @@
297
  "special": true
298
  },
299
  "32033": {
300
- "content": "<extra_id_69>",
301
  "lstrip": false,
302
  "normalized": false,
303
  "rstrip": false,
@@ -305,7 +305,7 @@
305
  "special": true
306
  },
307
  "32034": {
308
- "content": "<extra_id_14>",
309
  "lstrip": false,
310
  "normalized": false,
311
  "rstrip": false,
@@ -313,7 +313,7 @@
313
  "special": true
314
  },
315
  "32035": {
316
- "content": "<extra_id_65>",
317
  "lstrip": false,
318
  "normalized": false,
319
  "rstrip": false,
@@ -321,7 +321,7 @@
321
  "special": true
322
  },
323
  "32036": {
324
- "content": "<extra_id_88>",
325
  "lstrip": false,
326
  "normalized": false,
327
  "rstrip": false,
@@ -329,7 +329,7 @@
329
  "special": true
330
  },
331
  "32037": {
332
- "content": "<extra_id_97>",
333
  "lstrip": false,
334
  "normalized": false,
335
  "rstrip": false,
@@ -337,7 +337,7 @@
337
  "special": true
338
  },
339
  "32038": {
340
- "content": "<extra_id_94>",
341
  "lstrip": false,
342
  "normalized": false,
343
  "rstrip": false,
@@ -345,7 +345,7 @@
345
  "special": true
346
  },
347
  "32039": {
348
- "content": "<extra_id_27>",
349
  "lstrip": false,
350
  "normalized": false,
351
  "rstrip": false,
@@ -353,7 +353,7 @@
353
  "special": true
354
  },
355
  "32040": {
356
- "content": "<extra_id_80>",
357
  "lstrip": false,
358
  "normalized": false,
359
  "rstrip": false,
@@ -369,7 +369,7 @@
369
  "special": true
370
  },
371
  "32042": {
372
- "content": "<extra_id_55>",
373
  "lstrip": false,
374
  "normalized": false,
375
  "rstrip": false,
@@ -377,7 +377,7 @@
377
  "special": true
378
  },
379
  "32043": {
380
- "content": "<extra_id_68>",
381
  "lstrip": false,
382
  "normalized": false,
383
  "rstrip": false,
@@ -385,7 +385,7 @@
385
  "special": true
386
  },
387
  "32044": {
388
- "content": "<extra_id_98>",
389
  "lstrip": false,
390
  "normalized": false,
391
  "rstrip": false,
@@ -393,7 +393,7 @@
393
  "special": true
394
  },
395
  "32045": {
396
- "content": "<extra_id_24>",
397
  "lstrip": false,
398
  "normalized": false,
399
  "rstrip": false,
@@ -401,7 +401,7 @@
401
  "special": true
402
  },
403
  "32046": {
404
- "content": "<extra_id_22>",
405
  "lstrip": false,
406
  "normalized": false,
407
  "rstrip": false,
@@ -409,7 +409,7 @@
409
  "special": true
410
  },
411
  "32047": {
412
- "content": "<extra_id_49>",
413
  "lstrip": false,
414
  "normalized": false,
415
  "rstrip": false,
@@ -417,7 +417,7 @@
417
  "special": true
418
  },
419
  "32048": {
420
- "content": "<extra_id_91>",
421
  "lstrip": false,
422
  "normalized": false,
423
  "rstrip": false,
@@ -425,7 +425,7 @@
425
  "special": true
426
  },
427
  "32049": {
428
- "content": "<extra_id_64>",
429
  "lstrip": false,
430
  "normalized": false,
431
  "rstrip": false,
@@ -433,7 +433,7 @@
433
  "special": true
434
  },
435
  "32050": {
436
- "content": "<extra_id_99>",
437
  "lstrip": false,
438
  "normalized": false,
439
  "rstrip": false,
@@ -441,7 +441,7 @@
441
  "special": true
442
  },
443
  "32051": {
444
- "content": "<extra_id_13>",
445
  "lstrip": false,
446
  "normalized": false,
447
  "rstrip": false,
@@ -449,7 +449,7 @@
449
  "special": true
450
  },
451
  "32052": {
452
- "content": "<extra_id_45>",
453
  "lstrip": false,
454
  "normalized": false,
455
  "rstrip": false,
@@ -457,7 +457,7 @@
457
  "special": true
458
  },
459
  "32053": {
460
- "content": "<extra_id_29>",
461
  "lstrip": false,
462
  "normalized": false,
463
  "rstrip": false,
@@ -465,7 +465,7 @@
465
  "special": true
466
  },
467
  "32054": {
468
- "content": "<extra_id_11>",
469
  "lstrip": false,
470
  "normalized": false,
471
  "rstrip": false,
@@ -473,7 +473,7 @@
473
  "special": true
474
  },
475
  "32055": {
476
- "content": "<extra_id_87>",
477
  "lstrip": false,
478
  "normalized": false,
479
  "rstrip": false,
@@ -481,7 +481,7 @@
481
  "special": true
482
  },
483
  "32056": {
484
- "content": "<extra_id_47>",
485
  "lstrip": false,
486
  "normalized": false,
487
  "rstrip": false,
@@ -489,7 +489,7 @@
489
  "special": true
490
  },
491
  "32057": {
492
- "content": "<extra_id_37>",
493
  "lstrip": false,
494
  "normalized": false,
495
  "rstrip": false,
@@ -497,7 +497,7 @@
497
  "special": true
498
  },
499
  "32058": {
500
- "content": "<extra_id_86>",
501
  "lstrip": false,
502
  "normalized": false,
503
  "rstrip": false,
@@ -505,7 +505,7 @@
505
  "special": true
506
  },
507
  "32059": {
508
- "content": "<extra_id_63>",
509
  "lstrip": false,
510
  "normalized": false,
511
  "rstrip": false,
@@ -513,7 +513,7 @@
513
  "special": true
514
  },
515
  "32060": {
516
- "content": "<extra_id_43>",
517
  "lstrip": false,
518
  "normalized": false,
519
  "rstrip": false,
@@ -521,7 +521,7 @@
521
  "special": true
522
  },
523
  "32061": {
524
- "content": "<extra_id_25>",
525
  "lstrip": false,
526
  "normalized": false,
527
  "rstrip": false,
@@ -529,7 +529,7 @@
529
  "special": true
530
  },
531
  "32062": {
532
- "content": "<extra_id_21>",
533
  "lstrip": false,
534
  "normalized": false,
535
  "rstrip": false,
@@ -537,7 +537,7 @@
537
  "special": true
538
  },
539
  "32063": {
540
- "content": "<extra_id_7>",
541
  "lstrip": false,
542
  "normalized": false,
543
  "rstrip": false,
@@ -545,7 +545,7 @@
545
  "special": true
546
  },
547
  "32064": {
548
- "content": "<extra_id_92>",
549
  "lstrip": false,
550
  "normalized": false,
551
  "rstrip": false,
@@ -553,7 +553,7 @@
553
  "special": true
554
  },
555
  "32065": {
556
- "content": "<extra_id_56>",
557
  "lstrip": false,
558
  "normalized": false,
559
  "rstrip": false,
@@ -561,7 +561,7 @@
561
  "special": true
562
  },
563
  "32066": {
564
- "content": "<extra_id_17>",
565
  "lstrip": false,
566
  "normalized": false,
567
  "rstrip": false,
@@ -569,7 +569,7 @@
569
  "special": true
570
  },
571
  "32067": {
572
- "content": "<extra_id_9>",
573
  "lstrip": false,
574
  "normalized": false,
575
  "rstrip": false,
@@ -577,7 +577,7 @@
577
  "special": true
578
  },
579
  "32068": {
580
- "content": "<extra_id_81>",
581
  "lstrip": false,
582
  "normalized": false,
583
  "rstrip": false,
@@ -585,7 +585,7 @@
585
  "special": true
586
  },
587
  "32069": {
588
- "content": "<extra_id_20>",
589
  "lstrip": false,
590
  "normalized": false,
591
  "rstrip": false,
@@ -593,7 +593,7 @@
593
  "special": true
594
  },
595
  "32070": {
596
- "content": "<extra_id_30>",
597
  "lstrip": false,
598
  "normalized": false,
599
  "rstrip": false,
@@ -601,7 +601,7 @@
601
  "special": true
602
  },
603
  "32071": {
604
- "content": "<extra_id_44>",
605
  "lstrip": false,
606
  "normalized": false,
607
  "rstrip": false,
@@ -609,7 +609,7 @@
609
  "special": true
610
  },
611
  "32072": {
612
- "content": "<extra_id_77>",
613
  "lstrip": false,
614
  "normalized": false,
615
  "rstrip": false,
@@ -617,7 +617,7 @@
617
  "special": true
618
  },
619
  "32073": {
620
- "content": "<extra_id_89>",
621
  "lstrip": false,
622
  "normalized": false,
623
  "rstrip": false,
@@ -625,7 +625,7 @@
625
  "special": true
626
  },
627
  "32074": {
628
- "content": "<extra_id_52>",
629
  "lstrip": false,
630
  "normalized": false,
631
  "rstrip": false,
@@ -633,7 +633,7 @@
633
  "special": true
634
  },
635
  "32075": {
636
- "content": "<extra_id_83>",
637
  "lstrip": false,
638
  "normalized": false,
639
  "rstrip": false,
@@ -641,7 +641,7 @@
641
  "special": true
642
  },
643
  "32076": {
644
- "content": "<extra_id_85>",
645
  "lstrip": false,
646
  "normalized": false,
647
  "rstrip": false,
@@ -649,7 +649,7 @@
649
  "special": true
650
  },
651
  "32077": {
652
- "content": "<extra_id_75>",
653
  "lstrip": false,
654
  "normalized": false,
655
  "rstrip": false,
@@ -657,7 +657,7 @@
657
  "special": true
658
  },
659
  "32078": {
660
- "content": "<extra_id_0>",
661
  "lstrip": false,
662
  "normalized": false,
663
  "rstrip": false,
@@ -665,7 +665,7 @@
665
  "special": true
666
  },
667
  "32079": {
668
- "content": "<extra_id_12>",
669
  "lstrip": false,
670
  "normalized": false,
671
  "rstrip": false,
@@ -673,7 +673,7 @@
673
  "special": true
674
  },
675
  "32080": {
676
- "content": "<extra_id_54>",
677
  "lstrip": false,
678
  "normalized": false,
679
  "rstrip": false,
@@ -681,7 +681,7 @@
681
  "special": true
682
  },
683
  "32081": {
684
- "content": "<extra_id_62>",
685
  "lstrip": false,
686
  "normalized": false,
687
  "rstrip": false,
@@ -689,7 +689,7 @@
689
  "special": true
690
  },
691
  "32082": {
692
- "content": "<extra_id_76>",
693
  "lstrip": false,
694
  "normalized": false,
695
  "rstrip": false,
@@ -697,7 +697,7 @@
697
  "special": true
698
  },
699
  "32083": {
700
- "content": "<extra_id_70>",
701
  "lstrip": false,
702
  "normalized": false,
703
  "rstrip": false,
@@ -705,7 +705,7 @@
705
  "special": true
706
  },
707
  "32084": {
708
- "content": "<extra_id_28>",
709
  "lstrip": false,
710
  "normalized": false,
711
  "rstrip": false,
@@ -713,7 +713,7 @@
713
  "special": true
714
  },
715
  "32085": {
716
- "content": "<extra_id_8>",
717
  "lstrip": false,
718
  "normalized": false,
719
  "rstrip": false,
@@ -721,7 +721,7 @@
721
  "special": true
722
  },
723
  "32086": {
724
- "content": "<extra_id_84>",
725
  "lstrip": false,
726
  "normalized": false,
727
  "rstrip": false,
@@ -729,7 +729,7 @@
729
  "special": true
730
  },
731
  "32087": {
732
- "content": "<extra_id_2>",
733
  "lstrip": false,
734
  "normalized": false,
735
  "rstrip": false,
@@ -737,7 +737,7 @@
737
  "special": true
738
  },
739
  "32088": {
740
- "content": "<extra_id_61>",
741
  "lstrip": false,
742
  "normalized": false,
743
  "rstrip": false,
@@ -745,7 +745,7 @@
745
  "special": true
746
  },
747
  "32089": {
748
- "content": "<extra_id_10>",
749
  "lstrip": false,
750
  "normalized": false,
751
  "rstrip": false,
@@ -753,7 +753,7 @@
753
  "special": true
754
  },
755
  "32090": {
756
- "content": "<extra_id_53>",
757
  "lstrip": false,
758
  "normalized": false,
759
  "rstrip": false,
@@ -761,7 +761,7 @@
761
  "special": true
762
  },
763
  "32091": {
764
- "content": "<extra_id_6>",
765
  "lstrip": false,
766
  "normalized": false,
767
  "rstrip": false,
@@ -769,7 +769,7 @@
769
  "special": true
770
  },
771
  "32092": {
772
- "content": "<extra_id_67>",
773
  "lstrip": false,
774
  "normalized": false,
775
  "rstrip": false,
@@ -777,7 +777,7 @@
777
  "special": true
778
  },
779
  "32093": {
780
- "content": "<extra_id_71>",
781
  "lstrip": false,
782
  "normalized": false,
783
  "rstrip": false,
@@ -785,7 +785,7 @@
785
  "special": true
786
  },
787
  "32094": {
788
- "content": "<extra_id_66>",
789
  "lstrip": false,
790
  "normalized": false,
791
  "rstrip": false,
@@ -793,7 +793,7 @@
793
  "special": true
794
  },
795
  "32095": {
796
- "content": "<extra_id_46>",
797
  "lstrip": false,
798
  "normalized": false,
799
  "rstrip": false,
@@ -801,7 +801,7 @@
801
  "special": true
802
  },
803
  "32096": {
804
- "content": "<extra_id_18>",
805
  "lstrip": false,
806
  "normalized": false,
807
  "rstrip": false,
@@ -809,7 +809,7 @@
809
  "special": true
810
  },
811
  "32097": {
812
- "content": "<extra_id_38>",
813
  "lstrip": false,
814
  "normalized": false,
815
  "rstrip": false,
@@ -817,7 +817,7 @@
817
  "special": true
818
  },
819
  "32098": {
820
- "content": "<extra_id_59>",
821
  "lstrip": false,
822
  "normalized": false,
823
  "rstrip": false,
@@ -825,7 +825,7 @@
825
  "special": true
826
  },
827
  "32099": {
828
- "content": "<extra_id_40>",
829
  "lstrip": false,
830
  "normalized": false,
831
  "rstrip": false,
@@ -834,112 +834,112 @@
834
  }
835
  },
836
  "additional_special_tokens": [
837
- "<extra_id_39>",
838
- "<extra_id_95>",
839
- "<extra_id_93>",
840
- "<extra_id_79>",
841
- "<extra_id_32>",
842
- "<extra_id_15>",
843
- "<extra_id_73>",
844
- "<extra_id_36>",
845
  "<extra_id_1>",
846
- "<extra_id_19>",
847
- "<extra_id_26>",
848
- "<extra_id_60>",
849
- "<extra_id_51>",
850
- "<extra_id_5>",
851
  "<extra_id_3>",
852
- "<extra_id_31>",
853
- "<extra_id_58>",
854
- "<extra_id_23>",
855
- "<extra_id_50>",
856
- "<extra_id_78>",
857
- "<extra_id_96>",
858
- "<extra_id_72>",
859
- "<extra_id_74>",
860
  "<extra_id_4>",
861
- "<extra_id_48>",
862
- "<extra_id_42>",
863
- "<extra_id_90>",
864
- "<extra_id_82>",
865
- "<extra_id_57>",
866
- "<extra_id_34>",
867
- "<extra_id_16>",
868
- "<extra_id_33>",
869
- "<extra_id_35>",
870
- "<extra_id_69>",
871
  "<extra_id_14>",
872
- "<extra_id_65>",
873
- "<extra_id_88>",
874
- "<extra_id_97>",
875
- "<extra_id_94>",
876
- "<extra_id_27>",
877
- "<extra_id_80>",
878
- "<extra_id_41>",
879
- "<extra_id_55>",
880
- "<extra_id_68>",
881
- "<extra_id_98>",
882
- "<extra_id_24>",
883
  "<extra_id_22>",
884
- "<extra_id_49>",
885
- "<extra_id_91>",
886
- "<extra_id_64>",
887
- "<extra_id_99>",
888
- "<extra_id_13>",
889
- "<extra_id_45>",
890
  "<extra_id_29>",
891
- "<extra_id_11>",
892
- "<extra_id_87>",
893
- "<extra_id_47>",
 
 
 
 
894
  "<extra_id_37>",
895
- "<extra_id_86>",
896
- "<extra_id_63>",
 
 
 
897
  "<extra_id_43>",
898
- "<extra_id_25>",
899
- "<extra_id_21>",
900
- "<extra_id_7>",
901
- "<extra_id_92>",
902
- "<extra_id_56>",
903
- "<extra_id_17>",
904
- "<extra_id_9>",
905
- "<extra_id_81>",
906
- "<extra_id_20>",
907
- "<extra_id_30>",
908
  "<extra_id_44>",
909
- "<extra_id_77>",
910
- "<extra_id_89>",
 
 
 
 
 
911
  "<extra_id_52>",
912
- "<extra_id_83>",
913
- "<extra_id_85>",
914
- "<extra_id_75>",
915
- "<extra_id_0>",
916
- "<extra_id_12>",
917
  "<extra_id_54>",
918
- "<extra_id_62>",
919
- "<extra_id_76>",
920
- "<extra_id_70>",
921
- "<extra_id_28>",
922
- "<extra_id_8>",
923
- "<extra_id_84>",
924
- "<extra_id_2>",
925
  "<extra_id_61>",
926
- "<extra_id_10>",
927
- "<extra_id_53>",
928
- "<extra_id_6>",
 
 
929
  "<extra_id_67>",
 
 
 
930
  "<extra_id_71>",
931
- "<extra_id_66>",
932
- "<extra_id_46>",
933
- "<extra_id_18>",
934
- "<extra_id_38>",
935
- "<extra_id_59>",
936
- "<extra_id_40>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
937
  ],
938
  "bos_token": "<s>",
939
  "clean_up_tokenization_spaces": true,
940
  "eos_token": "</s>",
941
  "model_max_length": 1000000000000000019884624838656,
942
- "pad_token": "<pad>",
943
  "tokenizer_class": "PreTrainedTokenizerFast",
944
  "unk_token": "<unk>"
945
  }
 
33
  "special": true
34
  },
35
  "32000": {
36
+ "content": "<extra_id_0>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
 
41
  "special": true
42
  },
43
  "32001": {
44
+ "content": "<extra_id_1>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
 
49
  "special": true
50
  },
51
  "32002": {
52
+ "content": "<extra_id_2>",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
 
57
  "special": true
58
  },
59
  "32003": {
60
+ "content": "<extra_id_3>",
61
  "lstrip": false,
62
  "normalized": false,
63
  "rstrip": false,
 
65
  "special": true
66
  },
67
  "32004": {
68
+ "content": "<extra_id_4>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
 
73
  "special": true
74
  },
75
  "32005": {
76
+ "content": "<extra_id_5>",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
 
81
  "special": true
82
  },
83
  "32006": {
84
+ "content": "<extra_id_6>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
 
89
  "special": true
90
  },
91
  "32007": {
92
+ "content": "<extra_id_7>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
 
97
  "special": true
98
  },
99
  "32008": {
100
+ "content": "<extra_id_8>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
 
105
  "special": true
106
  },
107
  "32009": {
108
+ "content": "<extra_id_9>",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
 
113
  "special": true
114
  },
115
  "32010": {
116
+ "content": "<extra_id_10>",
117
  "lstrip": false,
118
  "normalized": false,
119
  "rstrip": false,
 
121
  "special": true
122
  },
123
  "32011": {
124
+ "content": "<extra_id_11>",
125
  "lstrip": false,
126
  "normalized": false,
127
  "rstrip": false,
 
129
  "special": true
130
  },
131
  "32012": {
132
+ "content": "<extra_id_12>",
133
  "lstrip": false,
134
  "normalized": false,
135
  "rstrip": false,
 
137
  "special": true
138
  },
139
  "32013": {
140
+ "content": "<extra_id_13>",
141
  "lstrip": false,
142
  "normalized": false,
143
  "rstrip": false,
 
145
  "special": true
146
  },
147
  "32014": {
148
+ "content": "<extra_id_14>",
149
  "lstrip": false,
150
  "normalized": false,
151
  "rstrip": false,
 
153
  "special": true
154
  },
155
  "32015": {
156
+ "content": "<extra_id_15>",
157
  "lstrip": false,
158
  "normalized": false,
159
  "rstrip": false,
 
161
  "special": true
162
  },
163
  "32016": {
164
+ "content": "<extra_id_16>",
165
  "lstrip": false,
166
  "normalized": false,
167
  "rstrip": false,
 
169
  "special": true
170
  },
171
  "32017": {
172
+ "content": "<extra_id_17>",
173
  "lstrip": false,
174
  "normalized": false,
175
  "rstrip": false,
 
177
  "special": true
178
  },
179
  "32018": {
180
+ "content": "<extra_id_18>",
181
  "lstrip": false,
182
  "normalized": false,
183
  "rstrip": false,
 
185
  "special": true
186
  },
187
  "32019": {
188
+ "content": "<extra_id_19>",
189
  "lstrip": false,
190
  "normalized": false,
191
  "rstrip": false,
 
193
  "special": true
194
  },
195
  "32020": {
196
+ "content": "<extra_id_20>",
197
  "lstrip": false,
198
  "normalized": false,
199
  "rstrip": false,
 
201
  "special": true
202
  },
203
  "32021": {
204
+ "content": "<extra_id_21>",
205
  "lstrip": false,
206
  "normalized": false,
207
  "rstrip": false,
 
209
  "special": true
210
  },
211
  "32022": {
212
+ "content": "<extra_id_22>",
213
  "lstrip": false,
214
  "normalized": false,
215
  "rstrip": false,
 
217
  "special": true
218
  },
219
  "32023": {
220
+ "content": "<extra_id_23>",
221
  "lstrip": false,
222
  "normalized": false,
223
  "rstrip": false,
 
225
  "special": true
226
  },
227
  "32024": {
228
+ "content": "<extra_id_24>",
229
  "lstrip": false,
230
  "normalized": false,
231
  "rstrip": false,
 
233
  "special": true
234
  },
235
  "32025": {
236
+ "content": "<extra_id_25>",
237
  "lstrip": false,
238
  "normalized": false,
239
  "rstrip": false,
 
241
  "special": true
242
  },
243
  "32026": {
244
+ "content": "<extra_id_26>",
245
  "lstrip": false,
246
  "normalized": false,
247
  "rstrip": false,
 
249
  "special": true
250
  },
251
  "32027": {
252
+ "content": "<extra_id_27>",
253
  "lstrip": false,
254
  "normalized": false,
255
  "rstrip": false,
 
257
  "special": true
258
  },
259
  "32028": {
260
+ "content": "<extra_id_28>",
261
  "lstrip": false,
262
  "normalized": false,
263
  "rstrip": false,
 
265
  "special": true
266
  },
267
  "32029": {
268
+ "content": "<extra_id_29>",
269
  "lstrip": false,
270
  "normalized": false,
271
  "rstrip": false,
 
273
  "special": true
274
  },
275
  "32030": {
276
+ "content": "<extra_id_30>",
277
  "lstrip": false,
278
  "normalized": false,
279
  "rstrip": false,
 
281
  "special": true
282
  },
283
  "32031": {
284
+ "content": "<extra_id_31>",
285
  "lstrip": false,
286
  "normalized": false,
287
  "rstrip": false,
 
289
  "special": true
290
  },
291
  "32032": {
292
+ "content": "<extra_id_32>",
293
  "lstrip": false,
294
  "normalized": false,
295
  "rstrip": false,
 
297
  "special": true
298
  },
299
  "32033": {
300
+ "content": "<extra_id_33>",
301
  "lstrip": false,
302
  "normalized": false,
303
  "rstrip": false,
 
305
  "special": true
306
  },
307
  "32034": {
308
+ "content": "<extra_id_34>",
309
  "lstrip": false,
310
  "normalized": false,
311
  "rstrip": false,
 
313
  "special": true
314
  },
315
  "32035": {
316
+ "content": "<extra_id_35>",
317
  "lstrip": false,
318
  "normalized": false,
319
  "rstrip": false,
 
321
  "special": true
322
  },
323
  "32036": {
324
+ "content": "<extra_id_36>",
325
  "lstrip": false,
326
  "normalized": false,
327
  "rstrip": false,
 
329
  "special": true
330
  },
331
  "32037": {
332
+ "content": "<extra_id_37>",
333
  "lstrip": false,
334
  "normalized": false,
335
  "rstrip": false,
 
337
  "special": true
338
  },
339
  "32038": {
340
+ "content": "<extra_id_38>",
341
  "lstrip": false,
342
  "normalized": false,
343
  "rstrip": false,
 
345
  "special": true
346
  },
347
  "32039": {
348
+ "content": "<extra_id_39>",
349
  "lstrip": false,
350
  "normalized": false,
351
  "rstrip": false,
 
353
  "special": true
354
  },
355
  "32040": {
356
+ "content": "<extra_id_40>",
357
  "lstrip": false,
358
  "normalized": false,
359
  "rstrip": false,
 
369
  "special": true
370
  },
371
  "32042": {
372
+ "content": "<extra_id_42>",
373
  "lstrip": false,
374
  "normalized": false,
375
  "rstrip": false,
 
377
  "special": true
378
  },
379
  "32043": {
380
+ "content": "<extra_id_43>",
381
  "lstrip": false,
382
  "normalized": false,
383
  "rstrip": false,
 
385
  "special": true
386
  },
387
  "32044": {
388
+ "content": "<extra_id_44>",
389
  "lstrip": false,
390
  "normalized": false,
391
  "rstrip": false,
 
393
  "special": true
394
  },
395
  "32045": {
396
+ "content": "<extra_id_45>",
397
  "lstrip": false,
398
  "normalized": false,
399
  "rstrip": false,
 
401
  "special": true
402
  },
403
  "32046": {
404
+ "content": "<extra_id_46>",
405
  "lstrip": false,
406
  "normalized": false,
407
  "rstrip": false,
 
409
  "special": true
410
  },
411
  "32047": {
412
+ "content": "<extra_id_47>",
413
  "lstrip": false,
414
  "normalized": false,
415
  "rstrip": false,
 
417
  "special": true
418
  },
419
  "32048": {
420
+ "content": "<extra_id_48>",
421
  "lstrip": false,
422
  "normalized": false,
423
  "rstrip": false,
 
425
  "special": true
426
  },
427
  "32049": {
428
+ "content": "<extra_id_49>",
429
  "lstrip": false,
430
  "normalized": false,
431
  "rstrip": false,
 
433
  "special": true
434
  },
435
  "32050": {
436
+ "content": "<extra_id_50>",
437
  "lstrip": false,
438
  "normalized": false,
439
  "rstrip": false,
 
441
  "special": true
442
  },
443
  "32051": {
444
+ "content": "<extra_id_51>",
445
  "lstrip": false,
446
  "normalized": false,
447
  "rstrip": false,
 
449
  "special": true
450
  },
451
  "32052": {
452
+ "content": "<extra_id_52>",
453
  "lstrip": false,
454
  "normalized": false,
455
  "rstrip": false,
 
457
  "special": true
458
  },
459
  "32053": {
460
+ "content": "<extra_id_53>",
461
  "lstrip": false,
462
  "normalized": false,
463
  "rstrip": false,
 
465
  "special": true
466
  },
467
  "32054": {
468
+ "content": "<extra_id_54>",
469
  "lstrip": false,
470
  "normalized": false,
471
  "rstrip": false,
 
473
  "special": true
474
  },
475
  "32055": {
476
+ "content": "<extra_id_55>",
477
  "lstrip": false,
478
  "normalized": false,
479
  "rstrip": false,
 
481
  "special": true
482
  },
483
  "32056": {
484
+ "content": "<extra_id_56>",
485
  "lstrip": false,
486
  "normalized": false,
487
  "rstrip": false,
 
489
  "special": true
490
  },
491
  "32057": {
492
+ "content": "<extra_id_57>",
493
  "lstrip": false,
494
  "normalized": false,
495
  "rstrip": false,
 
497
  "special": true
498
  },
499
  "32058": {
500
+ "content": "<extra_id_58>",
501
  "lstrip": false,
502
  "normalized": false,
503
  "rstrip": false,
 
505
  "special": true
506
  },
507
  "32059": {
508
+ "content": "<extra_id_59>",
509
  "lstrip": false,
510
  "normalized": false,
511
  "rstrip": false,
 
513
  "special": true
514
  },
515
  "32060": {
516
+ "content": "<extra_id_60>",
517
  "lstrip": false,
518
  "normalized": false,
519
  "rstrip": false,
 
521
  "special": true
522
  },
523
  "32061": {
524
+ "content": "<extra_id_61>",
525
  "lstrip": false,
526
  "normalized": false,
527
  "rstrip": false,
 
529
  "special": true
530
  },
531
  "32062": {
532
+ "content": "<extra_id_62>",
533
  "lstrip": false,
534
  "normalized": false,
535
  "rstrip": false,
 
537
  "special": true
538
  },
539
  "32063": {
540
+ "content": "<extra_id_63>",
541
  "lstrip": false,
542
  "normalized": false,
543
  "rstrip": false,
 
545
  "special": true
546
  },
547
  "32064": {
548
+ "content": "<extra_id_64>",
549
  "lstrip": false,
550
  "normalized": false,
551
  "rstrip": false,
 
553
  "special": true
554
  },
555
  "32065": {
556
+ "content": "<extra_id_65>",
557
  "lstrip": false,
558
  "normalized": false,
559
  "rstrip": false,
 
561
  "special": true
562
  },
563
  "32066": {
564
+ "content": "<extra_id_66>",
565
  "lstrip": false,
566
  "normalized": false,
567
  "rstrip": false,
 
569
  "special": true
570
  },
571
  "32067": {
572
+ "content": "<extra_id_67>",
573
  "lstrip": false,
574
  "normalized": false,
575
  "rstrip": false,
 
577
  "special": true
578
  },
579
  "32068": {
580
+ "content": "<extra_id_68>",
581
  "lstrip": false,
582
  "normalized": false,
583
  "rstrip": false,
 
585
  "special": true
586
  },
587
  "32069": {
588
+ "content": "<extra_id_69>",
589
  "lstrip": false,
590
  "normalized": false,
591
  "rstrip": false,
 
593
  "special": true
594
  },
595
  "32070": {
596
+ "content": "<extra_id_70>",
597
  "lstrip": false,
598
  "normalized": false,
599
  "rstrip": false,
 
601
  "special": true
602
  },
603
  "32071": {
604
+ "content": "<extra_id_71>",
605
  "lstrip": false,
606
  "normalized": false,
607
  "rstrip": false,
 
609
  "special": true
610
  },
611
  "32072": {
612
+ "content": "<extra_id_72>",
613
  "lstrip": false,
614
  "normalized": false,
615
  "rstrip": false,
 
617
  "special": true
618
  },
619
  "32073": {
620
+ "content": "<extra_id_73>",
621
  "lstrip": false,
622
  "normalized": false,
623
  "rstrip": false,
 
625
  "special": true
626
  },
627
  "32074": {
628
+ "content": "<extra_id_74>",
629
  "lstrip": false,
630
  "normalized": false,
631
  "rstrip": false,
 
633
  "special": true
634
  },
635
  "32075": {
636
+ "content": "<extra_id_75>",
637
  "lstrip": false,
638
  "normalized": false,
639
  "rstrip": false,
 
641
  "special": true
642
  },
643
  "32076": {
644
+ "content": "<extra_id_76>",
645
  "lstrip": false,
646
  "normalized": false,
647
  "rstrip": false,
 
649
  "special": true
650
  },
651
  "32077": {
652
+ "content": "<extra_id_77>",
653
  "lstrip": false,
654
  "normalized": false,
655
  "rstrip": false,
 
657
  "special": true
658
  },
659
  "32078": {
660
+ "content": "<extra_id_78>",
661
  "lstrip": false,
662
  "normalized": false,
663
  "rstrip": false,
 
665
  "special": true
666
  },
667
  "32079": {
668
+ "content": "<extra_id_79>",
669
  "lstrip": false,
670
  "normalized": false,
671
  "rstrip": false,
 
673
  "special": true
674
  },
675
  "32080": {
676
+ "content": "<extra_id_80>",
677
  "lstrip": false,
678
  "normalized": false,
679
  "rstrip": false,
 
681
  "special": true
682
  },
683
  "32081": {
684
+ "content": "<extra_id_81>",
685
  "lstrip": false,
686
  "normalized": false,
687
  "rstrip": false,
 
689
  "special": true
690
  },
691
  "32082": {
692
+ "content": "<extra_id_82>",
693
  "lstrip": false,
694
  "normalized": false,
695
  "rstrip": false,
 
697
  "special": true
698
  },
699
  "32083": {
700
+ "content": "<extra_id_83>",
701
  "lstrip": false,
702
  "normalized": false,
703
  "rstrip": false,
 
705
  "special": true
706
  },
707
  "32084": {
708
+ "content": "<extra_id_84>",
709
  "lstrip": false,
710
  "normalized": false,
711
  "rstrip": false,
 
713
  "special": true
714
  },
715
  "32085": {
716
+ "content": "<extra_id_85>",
717
  "lstrip": false,
718
  "normalized": false,
719
  "rstrip": false,
 
721
  "special": true
722
  },
723
  "32086": {
724
+ "content": "<extra_id_86>",
725
  "lstrip": false,
726
  "normalized": false,
727
  "rstrip": false,
 
729
  "special": true
730
  },
731
  "32087": {
732
+ "content": "<extra_id_87>",
733
  "lstrip": false,
734
  "normalized": false,
735
  "rstrip": false,
 
737
  "special": true
738
  },
739
  "32088": {
740
+ "content": "<extra_id_88>",
741
  "lstrip": false,
742
  "normalized": false,
743
  "rstrip": false,
 
745
  "special": true
746
  },
747
  "32089": {
748
+ "content": "<extra_id_89>",
749
  "lstrip": false,
750
  "normalized": false,
751
  "rstrip": false,
 
753
  "special": true
754
  },
755
  "32090": {
756
+ "content": "<extra_id_90>",
757
  "lstrip": false,
758
  "normalized": false,
759
  "rstrip": false,
 
761
  "special": true
762
  },
763
  "32091": {
764
+ "content": "<extra_id_91>",
765
  "lstrip": false,
766
  "normalized": false,
767
  "rstrip": false,
 
769
  "special": true
770
  },
771
  "32092": {
772
+ "content": "<extra_id_92>",
773
  "lstrip": false,
774
  "normalized": false,
775
  "rstrip": false,
 
777
  "special": true
778
  },
779
  "32093": {
780
+ "content": "<extra_id_93>",
781
  "lstrip": false,
782
  "normalized": false,
783
  "rstrip": false,
 
785
  "special": true
786
  },
787
  "32094": {
788
+ "content": "<extra_id_94>",
789
  "lstrip": false,
790
  "normalized": false,
791
  "rstrip": false,
 
793
  "special": true
794
  },
795
  "32095": {
796
+ "content": "<extra_id_95>",
797
  "lstrip": false,
798
  "normalized": false,
799
  "rstrip": false,
 
801
  "special": true
802
  },
803
  "32096": {
804
+ "content": "<extra_id_96>",
805
  "lstrip": false,
806
  "normalized": false,
807
  "rstrip": false,
 
809
  "special": true
810
  },
811
  "32097": {
812
+ "content": "<extra_id_97>",
813
  "lstrip": false,
814
  "normalized": false,
815
  "rstrip": false,
 
817
  "special": true
818
  },
819
  "32098": {
820
+ "content": "<extra_id_98>",
821
  "lstrip": false,
822
  "normalized": false,
823
  "rstrip": false,
 
825
  "special": true
826
  },
827
  "32099": {
828
+ "content": "<extra_id_99>",
829
  "lstrip": false,
830
  "normalized": false,
831
  "rstrip": false,
 
834
  }
835
  },
836
  "additional_special_tokens": [
837
+ "<extra_id_0>",
 
 
 
 
 
 
 
838
  "<extra_id_1>",
839
+ "<extra_id_2>",
 
 
 
 
840
  "<extra_id_3>",
 
 
 
 
 
 
 
 
841
  "<extra_id_4>",
842
+ "<extra_id_5>",
843
+ "<extra_id_6>",
844
+ "<extra_id_7>",
845
+ "<extra_id_8>",
846
+ "<extra_id_9>",
847
+ "<extra_id_10>",
848
+ "<extra_id_11>",
849
+ "<extra_id_12>",
850
+ "<extra_id_13>",
 
851
  "<extra_id_14>",
852
+ "<extra_id_15>",
853
+ "<extra_id_16>",
854
+ "<extra_id_17>",
855
+ "<extra_id_18>",
856
+ "<extra_id_19>",
857
+ "<extra_id_20>",
858
+ "<extra_id_21>",
 
 
 
 
859
  "<extra_id_22>",
860
+ "<extra_id_23>",
861
+ "<extra_id_24>",
862
+ "<extra_id_25>",
863
+ "<extra_id_26>",
864
+ "<extra_id_27>",
865
+ "<extra_id_28>",
866
  "<extra_id_29>",
867
+ "<extra_id_30>",
868
+ "<extra_id_31>",
869
+ "<extra_id_32>",
870
+ "<extra_id_33>",
871
+ "<extra_id_34>",
872
+ "<extra_id_35>",
873
+ "<extra_id_36>",
874
  "<extra_id_37>",
875
+ "<extra_id_38>",
876
+ "<extra_id_39>",
877
+ "<extra_id_40>",
878
+ "<extra_id_41>",
879
+ "<extra_id_42>",
880
  "<extra_id_43>",
 
 
 
 
 
 
 
 
 
 
881
  "<extra_id_44>",
882
+ "<extra_id_45>",
883
+ "<extra_id_46>",
884
+ "<extra_id_47>",
885
+ "<extra_id_48>",
886
+ "<extra_id_49>",
887
+ "<extra_id_50>",
888
+ "<extra_id_51>",
889
  "<extra_id_52>",
890
+ "<extra_id_53>",
 
 
 
 
891
  "<extra_id_54>",
892
+ "<extra_id_55>",
893
+ "<extra_id_56>",
894
+ "<extra_id_57>",
895
+ "<extra_id_58>",
896
+ "<extra_id_59>",
897
+ "<extra_id_60>",
 
898
  "<extra_id_61>",
899
+ "<extra_id_62>",
900
+ "<extra_id_63>",
901
+ "<extra_id_64>",
902
+ "<extra_id_65>",
903
+ "<extra_id_66>",
904
  "<extra_id_67>",
905
+ "<extra_id_68>",
906
+ "<extra_id_69>",
907
+ "<extra_id_70>",
908
  "<extra_id_71>",
909
+ "<extra_id_72>",
910
+ "<extra_id_73>",
911
+ "<extra_id_74>",
912
+ "<extra_id_75>",
913
+ "<extra_id_76>",
914
+ "<extra_id_77>",
915
+ "<extra_id_78>",
916
+ "<extra_id_79>",
917
+ "<extra_id_80>",
918
+ "<extra_id_81>",
919
+ "<extra_id_82>",
920
+ "<extra_id_83>",
921
+ "<extra_id_84>",
922
+ "<extra_id_85>",
923
+ "<extra_id_86>",
924
+ "<extra_id_87>",
925
+ "<extra_id_88>",
926
+ "<extra_id_89>",
927
+ "<extra_id_90>",
928
+ "<extra_id_91>",
929
+ "<extra_id_92>",
930
+ "<extra_id_93>",
931
+ "<extra_id_94>",
932
+ "<extra_id_95>",
933
+ "<extra_id_96>",
934
+ "<extra_id_97>",
935
+ "<extra_id_98>",
936
+ "<extra_id_99>"
937
  ],
938
  "bos_token": "<s>",
939
  "clean_up_tokenization_spaces": true,
940
  "eos_token": "</s>",
941
  "model_max_length": 1000000000000000019884624838656,
942
+ "pad_token": "<s>",
943
  "tokenizer_class": "PreTrainedTokenizerFast",
944
  "unk_token": "<unk>"
945
  }