uploading tokenizer
0c514c7
verified
|
{ |
|
"<mask>": 269397, |
|
"ace_Arab": 269195, |
|
"ace_Latn": 269196, |
|
"acm_Arab": 269197, |
|
"acq_Arab": 269198, |
|
"aeb_Arab": 269199, |
|
"afr_Latn": 269200, |
|
"ajp_Arab": 269201, |
|
"aka_Latn": 269202, |
|
"als_Latn": 269356, |
|
"amh_Ethi": 269203, |
|
"apc_Arab": 269204, |
|
"arb_Arab": 269205, |
|
"ars_Arab": 269206, |
|
"ary_Arab": 269207, |
|
"arz_Arab": 269208, |
|
"asm_Beng": 269209, |
|
"ast_Latn": 269210, |
|
"awa_Deva": 269211, |
|
"ayr_Latn": 269212, |
|
"azb_Arab": 269213, |
|
"azj_Latn": 269214, |
|
"bak_Cyrl": 269215, |
|
"bam_Latn": 269216, |
|
"ban_Latn": 269217, |
|
"bel_Cyrl": 269218, |
|
"bem_Latn": 269219, |
|
"ben_Beng": 269220, |
|
"bho_Deva": 269221, |
|
"bjn_Arab": 269222, |
|
"bjn_Latn": 269223, |
|
"bod_Tibt": 269224, |
|
"bos_Latn": 269225, |
|
"bug_Latn": 269226, |
|
"bul_Cyrl": 269227, |
|
"cat_Latn": 269228, |
|
"ceb_Latn": 269229, |
|
"ces_Latn": 269230, |
|
"cjk_Latn": 269231, |
|
"ckb_Arab": 269232, |
|
"crh_Latn": 269233, |
|
"cym_Latn": 269234, |
|
"dan_Latn": 269235, |
|
"deu_Latn": 269236, |
|
"dik_Latn": 269237, |
|
"dyu_Latn": 269238, |
|
"dzo_Tibt": 269239, |
|
"ell_Grek": 269240, |
|
"eng_Latn": 269241, |
|
"epo_Latn": 269242, |
|
"est_Latn": 269243, |
|
"eus_Latn": 269244, |
|
"ewe_Latn": 269245, |
|
"fao_Latn": 269246, |
|
"fij_Latn": 269248, |
|
"fin_Latn": 269249, |
|
"fon_Latn": 269250, |
|
"fra_Latn": 269251, |
|
"fur_Latn": 269252, |
|
"fuv_Latn": 269253, |
|
"gaz_Latn": 269329, |
|
"gla_Latn": 269254, |
|
"gle_Latn": 269255, |
|
"glg_Latn": 269256, |
|
"grn_Latn": 269257, |
|
"guj_Gujr": 269258, |
|
"hat_Latn": 269259, |
|
"hau_Latn": 269260, |
|
"heb_Hebr": 269261, |
|
"hin_Deva": 269262, |
|
"hne_Deva": 269263, |
|
"hrv_Latn": 269264, |
|
"hun_Latn": 269265, |
|
"hye_Armn": 269266, |
|
"ibo_Latn": 269267, |
|
"ilo_Latn": 269268, |
|
"ind_Latn": 269269, |
|
"isl_Latn": 269270, |
|
"ita_Latn": 269271, |
|
"jav_Latn": 269272, |
|
"jpn_Jpan": 269273, |
|
"kaa_Latn": 269398, |
|
"kab_Latn": 269274, |
|
"kac_Latn": 269275, |
|
"kam_Latn": 269276, |
|
"kan_Knda": 269277, |
|
"kas_Arab": 269278, |
|
"kas_Deva": 269279, |
|
"kat_Geor": 269280, |
|
"kaz_Cyrl": 269283, |
|
"kbp_Latn": 269284, |
|
"kea_Latn": 269285, |
|
"khk_Cyrl": 269316, |
|
"khm_Khmr": 269286, |
|
"kik_Latn": 269287, |
|
"kin_Latn": 269288, |
|
"kir_Cyrl": 269289, |
|
"kmb_Latn": 269290, |
|
"kmr_Latn": 269293, |
|
"knc_Arab": 269281, |
|
"knc_Latn": 269282, |
|
"kon_Latn": 269291, |
|
"kor_Hang": 269292, |
|
"lao_Laoo": 269294, |
|
"lij_Latn": 269296, |
|
"lim_Latn": 269297, |
|
"lin_Latn": 269298, |
|
"lit_Latn": 269299, |
|
"lmo_Latn": 269300, |
|
"ltg_Latn": 269301, |
|
"ltz_Latn": 269302, |
|
"lua_Latn": 269303, |
|
"lug_Latn": 269304, |
|
"luo_Latn": 269305, |
|
"lus_Latn": 269306, |
|
"lvs_Latn": 269295, |
|
"mag_Deva": 269307, |
|
"mai_Deva": 269308, |
|
"mal_Mlym": 269309, |
|
"mar_Deva": 269310, |
|
"min_Latn": 269311, |
|
"mkd_Cyrl": 269312, |
|
"mlt_Latn": 269314, |
|
"mni_Beng": 269315, |
|
"mos_Latn": 269317, |
|
"mri_Latn": 269318, |
|
"mya_Mymr": 269320, |
|
"nld_Latn": 269321, |
|
"nno_Latn": 269322, |
|
"nob_Latn": 269323, |
|
"npi_Deva": 269324, |
|
"nso_Latn": 269325, |
|
"nus_Latn": 269326, |
|
"nya_Latn": 269327, |
|
"oci_Latn": 269328, |
|
"ory_Orya": 269330, |
|
"pag_Latn": 269331, |
|
"pan_Guru": 269332, |
|
"pap_Latn": 269333, |
|
"pbt_Arab": 269337, |
|
"pes_Arab": 269247, |
|
"plt_Latn": 269313, |
|
"pol_Latn": 269334, |
|
"por_Latn": 269335, |
|
"prs_Arab": 269336, |
|
"quy_Latn": 269338, |
|
"ron_Latn": 269339, |
|
"run_Latn": 269340, |
|
"rus_Cyrl": 269341, |
|
"sag_Latn": 269342, |
|
"san_Deva": 269343, |
|
"sat_Beng": 269344, |
|
"scn_Latn": 269345, |
|
"shn_Mymr": 269346, |
|
"sin_Sinh": 269347, |
|
"slk_Latn": 269348, |
|
"slv_Latn": 269349, |
|
"smo_Latn": 269350, |
|
"sna_Latn": 269351, |
|
"snd_Arab": 269352, |
|
"som_Latn": 269353, |
|
"sot_Latn": 269354, |
|
"spa_Latn": 269355, |
|
"srd_Latn": 269357, |
|
"srp_Cyrl": 269358, |
|
"ssw_Latn": 269359, |
|
"sun_Latn": 269360, |
|
"swe_Latn": 269361, |
|
"swh_Latn": 269362, |
|
"szl_Latn": 269363, |
|
"tam_Taml": 269364, |
|
"taq_Latn": 269371, |
|
"taq_Tfng": 269372, |
|
"tat_Cyrl": 269365, |
|
"tel_Telu": 269366, |
|
"tgk_Cyrl": 269367, |
|
"tgl_Latn": 269368, |
|
"tha_Thai": 269369, |
|
"tir_Ethi": 269370, |
|
"tpi_Latn": 269373, |
|
"tsn_Latn": 269374, |
|
"tso_Latn": 269375, |
|
"tuk_Latn": 269376, |
|
"tum_Latn": 269377, |
|
"tur_Latn": 269378, |
|
"twi_Latn": 269379, |
|
"tzm_Tfng": 269380, |
|
"uig_Arab": 269381, |
|
"ukr_Cyrl": 269382, |
|
"umb_Latn": 269383, |
|
"urd_Arab": 269384, |
|
"uzn_Latn": 269385, |
|
"vec_Latn": 269386, |
|
"vie_Latn": 269387, |
|
"war_Latn": 269388, |
|
"wol_Latn": 269389, |
|
"xho_Latn": 269390, |
|
"ydd_Hebr": 269391, |
|
"yor_Latn": 269392, |
|
"yue_Hant": 269393, |
|
"zho_Hans": 269394, |
|
"zho_Hant": 269395, |
|
"zsm_Latn": 269319, |
|
"zul_Latn": 269396 |
|
} |
|
|