Spaces:
Sleeping
Sleeping
update
Browse files- examples/wenet/downaload_model.py +1 -1
- giga-tokens.txt +500 -0
- toolbox/k2_sherpa/nn_models.py +51 -7
examples/wenet/downaload_model.py
CHANGED
@@ -82,7 +82,7 @@ def main():
|
|
82 |
repo_id=args.repo_id,
|
83 |
filename=args.tokens_filename,
|
84 |
subfolder=args.tokens_sub_folder,
|
85 |
-
local_dir=
|
86 |
)
|
87 |
print(tokens_filename)
|
88 |
return
|
|
|
82 |
repo_id=args.repo_id,
|
83 |
filename=args.tokens_filename,
|
84 |
subfolder=args.tokens_sub_folder,
|
85 |
+
local_dir=local_model_dir.as_posix(),
|
86 |
)
|
87 |
print(tokens_filename)
|
88 |
return
|
giga-tokens.txt
ADDED
@@ -0,0 +1,500 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<blk> 0
|
2 |
+
<sos/eos> 1
|
3 |
+
<unk> 2
|
4 |
+
S 3
|
5 |
+
T 4
|
6 |
+
▁THE 5
|
7 |
+
▁A 6
|
8 |
+
E 7
|
9 |
+
▁AND 8
|
10 |
+
▁TO 9
|
11 |
+
N 10
|
12 |
+
D 11
|
13 |
+
▁OF 12
|
14 |
+
' 13
|
15 |
+
ING 14
|
16 |
+
▁I 15
|
17 |
+
Y 16
|
18 |
+
▁IN 17
|
19 |
+
ED 18
|
20 |
+
▁THAT 19
|
21 |
+
▁ 20
|
22 |
+
P 21
|
23 |
+
R 22
|
24 |
+
▁YOU 23
|
25 |
+
M 24
|
26 |
+
RE 25
|
27 |
+
ER 26
|
28 |
+
C 27
|
29 |
+
O 28
|
30 |
+
▁IT 29
|
31 |
+
L 30
|
32 |
+
A 31
|
33 |
+
U 32
|
34 |
+
G 33
|
35 |
+
▁WE 34
|
36 |
+
▁IS 35
|
37 |
+
▁SO 36
|
38 |
+
AL 37
|
39 |
+
I 38
|
40 |
+
▁S 39
|
41 |
+
▁RE 40
|
42 |
+
AR 41
|
43 |
+
B 42
|
44 |
+
▁FOR 43
|
45 |
+
▁C 44
|
46 |
+
▁BE 45
|
47 |
+
LE 46
|
48 |
+
F 47
|
49 |
+
W 48
|
50 |
+
▁E 49
|
51 |
+
▁HE 50
|
52 |
+
LL 51
|
53 |
+
▁WAS 52
|
54 |
+
LY 53
|
55 |
+
OR 54
|
56 |
+
IN 55
|
57 |
+
▁F 56
|
58 |
+
VE 57
|
59 |
+
▁THIS 58
|
60 |
+
TH 59
|
61 |
+
K 60
|
62 |
+
▁ON 61
|
63 |
+
IT 62
|
64 |
+
▁B 63
|
65 |
+
▁WITH 64
|
66 |
+
▁BUT 65
|
67 |
+
EN 66
|
68 |
+
CE 67
|
69 |
+
RI 68
|
70 |
+
▁DO 69
|
71 |
+
UR 70
|
72 |
+
▁HAVE 71
|
73 |
+
▁DE 72
|
74 |
+
▁ME 73
|
75 |
+
▁T 74
|
76 |
+
ENT 75
|
77 |
+
CH 76
|
78 |
+
▁THEY 77
|
79 |
+
▁NOT 78
|
80 |
+
ES 79
|
81 |
+
V 80
|
82 |
+
▁AS 81
|
83 |
+
RA 82
|
84 |
+
▁P 83
|
85 |
+
ON 84
|
86 |
+
TER 85
|
87 |
+
▁ARE 86
|
88 |
+
▁WHAT 87
|
89 |
+
IC 88
|
90 |
+
▁ST 89
|
91 |
+
▁LIKE 90
|
92 |
+
ATION 91
|
93 |
+
▁OR 92
|
94 |
+
▁CA 93
|
95 |
+
▁AT 94
|
96 |
+
H 95
|
97 |
+
▁KNOW 96
|
98 |
+
▁G 97
|
99 |
+
AN 98
|
100 |
+
▁CON 99
|
101 |
+
IL 100
|
102 |
+
ND 101
|
103 |
+
RO 102
|
104 |
+
▁HIS 103
|
105 |
+
▁CAN 104
|
106 |
+
▁ALL 105
|
107 |
+
TE 106
|
108 |
+
▁THERE 107
|
109 |
+
▁SU 108
|
110 |
+
▁MO 109
|
111 |
+
▁MA 110
|
112 |
+
LI 111
|
113 |
+
▁ONE 112
|
114 |
+
▁ABOUT 113
|
115 |
+
LA 114
|
116 |
+
▁CO 115
|
117 |
+
- 116
|
118 |
+
▁MY 117
|
119 |
+
▁HAD 118
|
120 |
+
CK 119
|
121 |
+
NG 120
|
122 |
+
▁NO 121
|
123 |
+
MENT 122
|
124 |
+
AD 123
|
125 |
+
LO 124
|
126 |
+
ME 125
|
127 |
+
▁AN 126
|
128 |
+
▁FROM 127
|
129 |
+
NE 128
|
130 |
+
▁IF 129
|
131 |
+
VER 130
|
132 |
+
▁JUST 131
|
133 |
+
▁PRO 132
|
134 |
+
ION 133
|
135 |
+
▁PA 134
|
136 |
+
▁WHO 135
|
137 |
+
▁SE 136
|
138 |
+
EL 137
|
139 |
+
IR 138
|
140 |
+
▁US 139
|
141 |
+
▁UP 140
|
142 |
+
▁YOUR 141
|
143 |
+
CI 142
|
144 |
+
RY 143
|
145 |
+
▁GO 144
|
146 |
+
▁SHE 145
|
147 |
+
▁LE 146
|
148 |
+
▁OUT 147
|
149 |
+
▁PO 148
|
150 |
+
▁HO 149
|
151 |
+
ATE 150
|
152 |
+
▁BO 151
|
153 |
+
▁BY 152
|
154 |
+
▁FA 153
|
155 |
+
▁MI 154
|
156 |
+
AS 155
|
157 |
+
MP 156
|
158 |
+
▁HER 157
|
159 |
+
VI 158
|
160 |
+
▁THINK 159
|
161 |
+
▁SOME 160
|
162 |
+
▁WHEN 161
|
163 |
+
▁AH 162
|
164 |
+
▁PEOPLE 163
|
165 |
+
IG 164
|
166 |
+
▁WA 165
|
167 |
+
▁TE 166
|
168 |
+
▁LA 167
|
169 |
+
▁WERE 168
|
170 |
+
▁LI 169
|
171 |
+
▁WOULD 170
|
172 |
+
▁SEE 171
|
173 |
+
▁WHICH 172
|
174 |
+
DE 173
|
175 |
+
GE 174
|
176 |
+
▁K 175
|
177 |
+
IGHT 176
|
178 |
+
▁HA 177
|
179 |
+
▁OUR 178
|
180 |
+
UN 179
|
181 |
+
▁HOW 180
|
182 |
+
▁GET 181
|
183 |
+
IS 182
|
184 |
+
UT 183
|
185 |
+
Z 184
|
186 |
+
CO 185
|
187 |
+
ET 186
|
188 |
+
UL 187
|
189 |
+
IES 188
|
190 |
+
IVE 189
|
191 |
+
AT 190
|
192 |
+
▁O 191
|
193 |
+
▁DON 192
|
194 |
+
LU 193
|
195 |
+
▁TIME 194
|
196 |
+
▁WILL 195
|
197 |
+
▁MORE 196
|
198 |
+
▁SP 197
|
199 |
+
▁NOW 198
|
200 |
+
RU 199
|
201 |
+
▁THEIR 200
|
202 |
+
▁UN 201
|
203 |
+
ITY 202
|
204 |
+
OL 203
|
205 |
+
X 204
|
206 |
+
TI 205
|
207 |
+
US 206
|
208 |
+
▁VERY 207
|
209 |
+
TION 208
|
210 |
+
▁FI 209
|
211 |
+
▁SAY 210
|
212 |
+
▁BECAUSE 211
|
213 |
+
▁EX 212
|
214 |
+
▁RO 213
|
215 |
+
ERS 214
|
216 |
+
IST 215
|
217 |
+
▁DA 216
|
218 |
+
TING 217
|
219 |
+
▁EN 218
|
220 |
+
OM 219
|
221 |
+
▁BA 220
|
222 |
+
▁BEEN 221
|
223 |
+
▁LO 222
|
224 |
+
▁UM 223
|
225 |
+
AGE 224
|
226 |
+
ABLE 225
|
227 |
+
▁WO 226
|
228 |
+
▁RA 227
|
229 |
+
▁OTHER 228
|
230 |
+
▁REALLY 229
|
231 |
+
ENCE 230
|
232 |
+
▁GOING 231
|
233 |
+
▁HIM 232
|
234 |
+
▁HAS 233
|
235 |
+
▁THEM 234
|
236 |
+
▁DIS 235
|
237 |
+
▁WANT 236
|
238 |
+
ID 237
|
239 |
+
TA 238
|
240 |
+
▁LOOK 239
|
241 |
+
KE 240
|
242 |
+
▁DID 241
|
243 |
+
▁SA 242
|
244 |
+
▁VI 243
|
245 |
+
▁SAID 244
|
246 |
+
▁RIGHT 245
|
247 |
+
▁THESE 246
|
248 |
+
▁WORK 247
|
249 |
+
▁COM 248
|
250 |
+
ALLY 249
|
251 |
+
FF 250
|
252 |
+
QU 251
|
253 |
+
AC 252
|
254 |
+
▁DR 253
|
255 |
+
▁WAY 254
|
256 |
+
▁INTO 255
|
257 |
+
MO 256
|
258 |
+
TED 257
|
259 |
+
EST 258
|
260 |
+
▁HERE 259
|
261 |
+
OK 260
|
262 |
+
▁COULD 261
|
263 |
+
▁WELL 262
|
264 |
+
MA 263
|
265 |
+
▁PRE 264
|
266 |
+
▁DI 265
|
267 |
+
MAN 266
|
268 |
+
▁COMP 267
|
269 |
+
▁THEN 268
|
270 |
+
IM 269
|
271 |
+
▁PER 270
|
272 |
+
▁NA 271
|
273 |
+
▁WHERE 272
|
274 |
+
▁TWO 273
|
275 |
+
▁WI 274
|
276 |
+
▁FE 275
|
277 |
+
INE 276
|
278 |
+
▁ANY 277
|
279 |
+
TURE 278
|
280 |
+
▁OVER 279
|
281 |
+
BO 280
|
282 |
+
ACH 281
|
283 |
+
OW 282
|
284 |
+
▁MAKE 283
|
285 |
+
▁TRA 284
|
286 |
+
HE 285
|
287 |
+
UND 286
|
288 |
+
▁EVEN 287
|
289 |
+
ANCE 288
|
290 |
+
▁YEAR 289
|
291 |
+
HO 290
|
292 |
+
AM 291
|
293 |
+
▁CHA 292
|
294 |
+
▁BACK 293
|
295 |
+
VO 294
|
296 |
+
ANT 295
|
297 |
+
DI 296
|
298 |
+
▁ALSO 297
|
299 |
+
▁THOSE 298
|
300 |
+
▁MAN 299
|
301 |
+
CTION 300
|
302 |
+
ICAL 301
|
303 |
+
▁JO 302
|
304 |
+
▁OP 303
|
305 |
+
▁NEW 304
|
306 |
+
▁MU 305
|
307 |
+
▁HU 306
|
308 |
+
▁KIND 307
|
309 |
+
▁NE 308
|
310 |
+
CA 309
|
311 |
+
END 310
|
312 |
+
TIC 311
|
313 |
+
FUL 312
|
314 |
+
▁YEAH 313
|
315 |
+
SH 314
|
316 |
+
▁APP 315
|
317 |
+
▁THINGS 316
|
318 |
+
SIDE 317
|
319 |
+
▁GOOD 318
|
320 |
+
ONE 319
|
321 |
+
▁TAKE 320
|
322 |
+
CU 321
|
323 |
+
▁EVERY 322
|
324 |
+
▁MEAN 323
|
325 |
+
▁FIRST 324
|
326 |
+
OP 325
|
327 |
+
▁TH 326
|
328 |
+
▁MUCH 327
|
329 |
+
▁PART 328
|
330 |
+
UGH 329
|
331 |
+
▁COME 330
|
332 |
+
J 331
|
333 |
+
▁THAN 332
|
334 |
+
▁EXP 333
|
335 |
+
▁AGAIN 334
|
336 |
+
▁LITTLE 335
|
337 |
+
MB 336
|
338 |
+
▁NEED 337
|
339 |
+
▁TALK 338
|
340 |
+
IF 339
|
341 |
+
FOR 340
|
342 |
+
▁SH 341
|
343 |
+
ISH 342
|
344 |
+
▁STA 343
|
345 |
+
ATED 344
|
346 |
+
▁GU 345
|
347 |
+
▁LET 346
|
348 |
+
IA 347
|
349 |
+
▁MAR 348
|
350 |
+
▁DOWN 349
|
351 |
+
▁DAY 350
|
352 |
+
▁GA 351
|
353 |
+
▁SOMETHING 352
|
354 |
+
▁BU 353
|
355 |
+
DUC 354
|
356 |
+
HA 355
|
357 |
+
▁LOT 356
|
358 |
+
▁RU 357
|
359 |
+
▁THOUGH 358
|
360 |
+
▁GREAT 359
|
361 |
+
AIN 360
|
362 |
+
▁THROUGH 361
|
363 |
+
▁THING 362
|
364 |
+
OUS 363
|
365 |
+
▁PRI 364
|
366 |
+
▁GOT 365
|
367 |
+
▁SHOULD 366
|
368 |
+
▁AFTER 367
|
369 |
+
▁HEAR 368
|
370 |
+
▁TA 369
|
371 |
+
▁ONLY 370
|
372 |
+
▁CHI 371
|
373 |
+
IOUS 372
|
374 |
+
▁SHA 373
|
375 |
+
▁MOST 374
|
376 |
+
▁ACTUALLY 375
|
377 |
+
▁START 376
|
378 |
+
LIC 377
|
379 |
+
▁VA 378
|
380 |
+
▁RI 379
|
381 |
+
DAY 380
|
382 |
+
IAN 381
|
383 |
+
▁DOES 382
|
384 |
+
ROW 383
|
385 |
+
▁GRA 384
|
386 |
+
ITION 385
|
387 |
+
▁MANY 386
|
388 |
+
▁BEFORE 387
|
389 |
+
▁GIVE 388
|
390 |
+
PORT 389
|
391 |
+
QUI 390
|
392 |
+
▁LIFE 391
|
393 |
+
▁WORLD 392
|
394 |
+
▁PI 393
|
395 |
+
▁LONG 394
|
396 |
+
▁THREE 395
|
397 |
+
IZE 396
|
398 |
+
NESS 397
|
399 |
+
▁SHOW 398
|
400 |
+
PH 399
|
401 |
+
▁WHY 400
|
402 |
+
▁QUESTION 401
|
403 |
+
WARD 402
|
404 |
+
▁THANK 403
|
405 |
+
▁PH 404
|
406 |
+
▁DIFFERENT 405
|
407 |
+
▁OWN 406
|
408 |
+
▁FEEL 407
|
409 |
+
▁MIGHT 408
|
410 |
+
▁HAPPEN 409
|
411 |
+
▁MADE 410
|
412 |
+
▁BRO 411
|
413 |
+
IBLE 412
|
414 |
+
▁HI 413
|
415 |
+
▁STATE 414
|
416 |
+
▁HAND 415
|
417 |
+
▁NEVER 416
|
418 |
+
▁PLACE 417
|
419 |
+
▁LOVE 418
|
420 |
+
▁DU 419
|
421 |
+
▁POINT 420
|
422 |
+
▁HELP 421
|
423 |
+
▁COUNT 422
|
424 |
+
▁STILL 423
|
425 |
+
▁MR 424
|
426 |
+
▁FIND 425
|
427 |
+
▁PERSON 426
|
428 |
+
▁CAME 427
|
429 |
+
▁SAME 428
|
430 |
+
▁LAST 429
|
431 |
+
▁HIGH 430
|
432 |
+
▁OLD 431
|
433 |
+
▁UNDER 432
|
434 |
+
▁FOUR 433
|
435 |
+
▁AROUND 434
|
436 |
+
▁SORT 435
|
437 |
+
▁CHANGE 436
|
438 |
+
▁YES 437
|
439 |
+
SHIP 438
|
440 |
+
▁ANOTHER 439
|
441 |
+
ATIVE 440
|
442 |
+
▁FOUND 441
|
443 |
+
▁JA 442
|
444 |
+
▁ALWAYS 443
|
445 |
+
▁NEXT 444
|
446 |
+
▁TURN 445
|
447 |
+
▁JU 446
|
448 |
+
▁SIX 447
|
449 |
+
▁FACT 448
|
450 |
+
▁INTEREST 449
|
451 |
+
▁WORD 450
|
452 |
+
▁THOUSAND 451
|
453 |
+
▁HUNDRED 452
|
454 |
+
▁NUMBER 453
|
455 |
+
▁IDEA 454
|
456 |
+
▁PLAN 455
|
457 |
+
▁COURSE 456
|
458 |
+
▁SCHOOL 457
|
459 |
+
▁HOUSE 458
|
460 |
+
▁TWENTY 459
|
461 |
+
▁JE 460
|
462 |
+
▁PLAY 461
|
463 |
+
▁AWAY 462
|
464 |
+
▁LEARN 463
|
465 |
+
▁HARD 464
|
466 |
+
▁WEEK 465
|
467 |
+
▁BETTER 466
|
468 |
+
▁WHILE 467
|
469 |
+
▁FRIEND 468
|
470 |
+
▁OKAY 469
|
471 |
+
▁NINE 470
|
472 |
+
▁UNDERSTAND 471
|
473 |
+
▁KEEP 472
|
474 |
+
▁GONNA 473
|
475 |
+
▁SYSTEM 474
|
476 |
+
▁AMERICA 475
|
477 |
+
▁POWER 476
|
478 |
+
▁IMPORTANT 477
|
479 |
+
▁WITHOUT 478
|
480 |
+
▁MAYBE 479
|
481 |
+
▁SEVEN 480
|
482 |
+
▁BETWEEN 481
|
483 |
+
▁BUILD 482
|
484 |
+
▁CERTAIN 483
|
485 |
+
▁PROBLEM 484
|
486 |
+
▁MONEY 485
|
487 |
+
▁BELIEVE 486
|
488 |
+
▁SECOND 487
|
489 |
+
▁REASON 488
|
490 |
+
▁TOGETHER 489
|
491 |
+
▁PUBLIC 490
|
492 |
+
▁ANYTHING 491
|
493 |
+
▁SPEAK 492
|
494 |
+
▁BUSINESS 493
|
495 |
+
▁EVERYTHING 494
|
496 |
+
▁CLOSE 495
|
497 |
+
▁QUITE 496
|
498 |
+
▁ANSWER 497
|
499 |
+
▁ENOUGH 498
|
500 |
+
Q 499
|
toolbox/k2_sherpa/nn_models.py
CHANGED
@@ -160,6 +160,26 @@ model_map = {
|
|
160 |
"tokens_file_sub_folder": ".",
|
161 |
"loader": "load_sherpa_offline_recognizer_from_whisper",
|
162 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
163 |
{
|
164 |
"repo_id": "csukuangfj/sherpa-onnx-paraformer-en-2024-03-09",
|
165 |
"nn_model_file": "model.int8.onnx",
|
@@ -168,6 +188,26 @@ model_map = {
|
|
168 |
"tokens_file_sub_folder": ".",
|
169 |
"loader": "load_sherpa_offline_recognizer_from_paraformer",
|
170 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
171 |
],
|
172 |
"Chinese+English": [
|
173 |
{
|
@@ -235,12 +275,14 @@ def download_model(local_model_dir: str,
|
|
235 |
|
236 |
if "tokens_file" in kwargs.keys():
|
237 |
main_logger.info("download tokens_file. filename: {}, subfolder: {}".format(kwargs["tokens_file"], kwargs["tokens_file_sub_folder"]))
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
|
|
|
|
244 |
|
245 |
|
246 |
def load_sherpa_offline_recognizer(nn_model_file: str,
|
@@ -358,7 +400,9 @@ def load_recognizer(local_model_dir: Path,
|
|
358 |
joiner_model_file = (local_model_dir / kwargs["joiner_model_file_sub_folder"] / kwargs["joiner_model_file"]).as_posix()
|
359 |
kwargs_["joiner_model_file"] = joiner_model_file
|
360 |
if "tokens_file" in kwargs.keys():
|
361 |
-
tokens_file =
|
|
|
|
|
362 |
kwargs_["tokens_file"] = tokens_file
|
363 |
if "normalize_samples" in kwargs.keys():
|
364 |
kwargs_["normalize_samples"] = kwargs["normalize_samples"]
|
|
|
160 |
"tokens_file_sub_folder": ".",
|
161 |
"loader": "load_sherpa_offline_recognizer_from_whisper",
|
162 |
},
|
163 |
+
{
|
164 |
+
"repo_id": "csukuangfj/sherpa-onnx-whisper-base.en",
|
165 |
+
"encoder_model_file": "base.en-encoder.int8.onnx",
|
166 |
+
"encoder_model_file_sub_folder": ".",
|
167 |
+
"decoder_model_file": "base.en-decoder.int8.onnx",
|
168 |
+
"decoder_model_file_sub_folder": ".",
|
169 |
+
"tokens_file": "base.en-tokens.txt",
|
170 |
+
"tokens_file_sub_folder": ".",
|
171 |
+
"loader": "load_sherpa_offline_recognizer_from_whisper",
|
172 |
+
},
|
173 |
+
{
|
174 |
+
"repo_id": "csukuangfj/sherpa-onnx-whisper-small.en",
|
175 |
+
"encoder_model_file": "small.en-encoder.int8.onnx",
|
176 |
+
"encoder_model_file_sub_folder": ".",
|
177 |
+
"decoder_model_file": "small.en-decoder.int8.onnx",
|
178 |
+
"decoder_model_file_sub_folder": ".",
|
179 |
+
"tokens_file": "small.en-tokens.txt",
|
180 |
+
"tokens_file_sub_folder": ".",
|
181 |
+
"loader": "load_sherpa_offline_recognizer_from_whisper",
|
182 |
+
},
|
183 |
{
|
184 |
"repo_id": "csukuangfj/sherpa-onnx-paraformer-en-2024-03-09",
|
185 |
"nn_model_file": "model.int8.onnx",
|
|
|
188 |
"tokens_file_sub_folder": ".",
|
189 |
"loader": "load_sherpa_offline_recognizer_from_paraformer",
|
190 |
},
|
191 |
+
{
|
192 |
+
"repo_id": "yfyeung/icefall-asr-gigaspeech-zipformer-2023-10-17",
|
193 |
+
"encoder_model_file": "encoder-epoch-30-avg-9.onnx",
|
194 |
+
"encoder_model_file_sub_folder": "exp",
|
195 |
+
"decoder_model_file": "decoder-epoch-30-avg-9.onnx",
|
196 |
+
"decoder_model_file_sub_folder": "exp",
|
197 |
+
"joiner_model_file": "joiner-epoch-30-avg-9.onnx",
|
198 |
+
"joiner_model_file_sub_folder": "exp",
|
199 |
+
"tokens_file": "tokens.txt",
|
200 |
+
"tokens_file_sub_folder": "data/lang_bpe_500",
|
201 |
+
"loader": "load_sherpa_offline_recognizer_from_transducer",
|
202 |
+
},
|
203 |
+
{
|
204 |
+
"repo_id": "wgb14/icefall-asr-gigaspeech-pruned-transducer-stateless2",
|
205 |
+
"nn_model_file": "cpu_jit-iter-3488000-avg-20.pt",
|
206 |
+
"nn_model_file_sub_folder": "exp",
|
207 |
+
"tokens_file": "./giga-tokens.txt",
|
208 |
+
"tokens_file_sub_folder": ".",
|
209 |
+
"loader": "load_sherpa_offline_recognizer",
|
210 |
+
},
|
211 |
],
|
212 |
"Chinese+English": [
|
213 |
{
|
|
|
275 |
|
276 |
if "tokens_file" in kwargs.keys():
|
277 |
main_logger.info("download tokens_file. filename: {}, subfolder: {}".format(kwargs["tokens_file"], kwargs["tokens_file_sub_folder"]))
|
278 |
+
tokens_file = kwargs["tokens_file"]
|
279 |
+
if not tokens_file.startswith("./"):
|
280 |
+
_ = huggingface_hub.hf_hub_download(
|
281 |
+
repo_id=repo_id,
|
282 |
+
filename=kwargs["tokens_file"],
|
283 |
+
subfolder=kwargs["tokens_file_sub_folder"],
|
284 |
+
local_dir=local_model_dir,
|
285 |
+
)
|
286 |
|
287 |
|
288 |
def load_sherpa_offline_recognizer(nn_model_file: str,
|
|
|
400 |
joiner_model_file = (local_model_dir / kwargs["joiner_model_file_sub_folder"] / kwargs["joiner_model_file"]).as_posix()
|
401 |
kwargs_["joiner_model_file"] = joiner_model_file
|
402 |
if "tokens_file" in kwargs.keys():
|
403 |
+
tokens_file: str = kwargs["tokens_file"]
|
404 |
+
if not tokens_file.startswith("./"):
|
405 |
+
tokens_file = (local_model_dir / kwargs["tokens_file_sub_folder"] / kwargs["tokens_file"]).as_posix()
|
406 |
kwargs_["tokens_file"] = tokens_file
|
407 |
if "normalize_samples" in kwargs.keys():
|
408 |
kwargs_["normalize_samples"] = kwargs["normalize_samples"]
|