llama3-s-2024-07-08 / _special_tokens_maps.json
jan-hq's picture
Upload new tokenizer.model and special token map dict
f66c9a1 verified
raw
history blame
10.6 kB
{"<|begin_of_text|>": 128000, "<|end_of_text|>": 128001, "<|reserved_special_token_0|>": 128002, "<|reserved_special_token_1|>": 128003, "<|reserved_special_token_2|>": 128004, "<|reserved_special_token_3|>": 128005, "<|start_header_id|>": 128006, "<|end_header_id|>": 128007, "<|reserved_special_token_4|>": 128008, "<|eot_id|>": 128009, "<|reserved_special_token_5|>": 128010, "<|reserved_special_token_6|>": 128011, "<|reserved_special_token_7|>": 128012, "<|reserved_special_token_8|>": 128013, "<|reserved_special_token_9|>": 128014, "<|reserved_special_token_10|>": 128015, "<|reserved_special_token_11|>": 128016, "<|reserved_special_token_12|>": 128017, "<|reserved_special_token_13|>": 128018, "<|reserved_special_token_14|>": 128019, "<|reserved_special_token_15|>": 128020, "<|reserved_special_token_16|>": 128021, "<|reserved_special_token_17|>": 128022, "<|reserved_special_token_18|>": 128023, "<|reserved_special_token_19|>": 128024, "<|reserved_special_token_20|>": 128025, "<|reserved_special_token_21|>": 128026, "<|reserved_special_token_22|>": 128027, "<|reserved_special_token_23|>": 128028, "<|reserved_special_token_24|>": 128029, "<|reserved_special_token_25|>": 128030, "<|reserved_special_token_26|>": 128031, "<|reserved_special_token_27|>": 128032, "<|reserved_special_token_28|>": 128033, "<|reserved_special_token_29|>": 128034, "<|reserved_special_token_30|>": 128035, "<|reserved_special_token_31|>": 128036, "<|reserved_special_token_32|>": 128037, "<|reserved_special_token_33|>": 128038, "<|reserved_special_token_34|>": 128039, "<|reserved_special_token_35|>": 128040, "<|reserved_special_token_36|>": 128041, "<|reserved_special_token_37|>": 128042, "<|reserved_special_token_38|>": 128043, "<|reserved_special_token_39|>": 128044, "<|reserved_special_token_40|>": 128045, "<|reserved_special_token_41|>": 128046, "<|reserved_special_token_42|>": 128047, "<|reserved_special_token_43|>": 128048, "<|reserved_special_token_44|>": 128049, "<|reserved_special_token_45|>": 128050, "<|reserved_special_token_46|>": 128051, "<|reserved_special_token_47|>": 128052, "<|reserved_special_token_48|>": 128053, "<|reserved_special_token_49|>": 128054, "<|reserved_special_token_50|>": 128055, "<|reserved_special_token_51|>": 128056, "<|reserved_special_token_52|>": 128057, "<|reserved_special_token_53|>": 128058, "<|reserved_special_token_54|>": 128059, "<|reserved_special_token_55|>": 128060, "<|reserved_special_token_56|>": 128061, "<|reserved_special_token_57|>": 128062, "<|reserved_special_token_58|>": 128063, "<|reserved_special_token_59|>": 128064, "<|reserved_special_token_60|>": 128065, "<|reserved_special_token_61|>": 128066, "<|reserved_special_token_62|>": 128067, "<|reserved_special_token_63|>": 128068, "<|reserved_special_token_64|>": 128069, "<|reserved_special_token_65|>": 128070, "<|reserved_special_token_66|>": 128071, "<|reserved_special_token_67|>": 128072, "<|reserved_special_token_68|>": 128073, "<|reserved_special_token_69|>": 128074, "<|reserved_special_token_70|>": 128075, "<|reserved_special_token_71|>": 128076, "<|reserved_special_token_72|>": 128077, "<|reserved_special_token_73|>": 128078, "<|reserved_special_token_74|>": 128079, "<|reserved_special_token_75|>": 128080, "<|reserved_special_token_76|>": 128081, "<|reserved_special_token_77|>": 128082, "<|reserved_special_token_78|>": 128083, "<|reserved_special_token_79|>": 128084, "<|reserved_special_token_80|>": 128085, "<|reserved_special_token_81|>": 128086, "<|reserved_special_token_82|>": 128087, "<|reserved_special_token_83|>": 128088, "<|reserved_special_token_84|>": 128089, "<|reserved_special_token_85|>": 128090, "<|reserved_special_token_86|>": 128091, "<|reserved_special_token_87|>": 128092, "<|reserved_special_token_88|>": 128093, "<|reserved_special_token_89|>": 128094, "<|reserved_special_token_90|>": 128095, "<|reserved_special_token_91|>": 128096, "<|reserved_special_token_92|>": 128097, "<|reserved_special_token_93|>": 128098, "<|reserved_special_token_94|>": 128099, "<|reserved_special_token_95|>": 128100, "<|reserved_special_token_96|>": 128101, "<|reserved_special_token_97|>": 128102, "<|reserved_special_token_98|>": 128103, "<|reserved_special_token_99|>": 128104, "<|reserved_special_token_100|>": 128105, "<|reserved_special_token_101|>": 128106, "<|reserved_special_token_102|>": 128107, "<|reserved_special_token_103|>": 128108, "<|reserved_special_token_104|>": 128109, "<|reserved_special_token_105|>": 128110, "<|reserved_special_token_106|>": 128111, "<|reserved_special_token_107|>": 128112, "<|reserved_special_token_108|>": 128113, "<|reserved_special_token_109|>": 128114, "<|reserved_special_token_110|>": 128115, "<|reserved_special_token_111|>": 128116, "<|reserved_special_token_112|>": 128117, "<|reserved_special_token_113|>": 128118, "<|reserved_special_token_114|>": 128119, "<|reserved_special_token_115|>": 128120, "<|reserved_special_token_116|>": 128121, "<|reserved_special_token_117|>": 128122, "<|reserved_special_token_118|>": 128123, "<|reserved_special_token_119|>": 128124, "<|reserved_special_token_120|>": 128125, "<|reserved_special_token_121|>": 128126, "<|reserved_special_token_122|>": 128127, "<|reserved_special_token_123|>": 128128, "<|reserved_special_token_124|>": 128129, "<|reserved_special_token_125|>": 128130, "<|reserved_special_token_126|>": 128131, "<|reserved_special_token_127|>": 128132, "<|reserved_special_token_128|>": 128133, "<|reserved_special_token_129|>": 128134, "<|reserved_special_token_130|>": 128135, "<|reserved_special_token_131|>": 128136, "<|reserved_special_token_132|>": 128137, "<|reserved_special_token_133|>": 128138, "<|reserved_special_token_134|>": 128139, "<|reserved_special_token_135|>": 128140, "<|reserved_special_token_136|>": 128141, "<|reserved_special_token_137|>": 128142, "<|reserved_special_token_138|>": 128143, "<|reserved_special_token_139|>": 128144, "<|reserved_special_token_140|>": 128145, "<|reserved_special_token_141|>": 128146, "<|reserved_special_token_142|>": 128147, "<|reserved_special_token_143|>": 128148, "<|reserved_special_token_144|>": 128149, "<|reserved_special_token_145|>": 128150, "<|reserved_special_token_146|>": 128151, "<|reserved_special_token_147|>": 128152, "<|reserved_special_token_148|>": 128153, "<|reserved_special_token_149|>": 128154, "<|reserved_special_token_150|>": 128155, "<|reserved_special_token_151|>": 128156, "<|reserved_special_token_152|>": 128157, "<|reserved_special_token_153|>": 128158, "<|reserved_special_token_154|>": 128159, "<|reserved_special_token_155|>": 128160, "<|reserved_special_token_156|>": 128161, "<|reserved_special_token_157|>": 128162, "<|reserved_special_token_158|>": 128163, "<|reserved_special_token_159|>": 128164, "<|reserved_special_token_160|>": 128165, "<|reserved_special_token_161|>": 128166, "<|reserved_special_token_162|>": 128167, "<|reserved_special_token_163|>": 128168, "<|reserved_special_token_164|>": 128169, "<|reserved_special_token_165|>": 128170, "<|reserved_special_token_166|>": 128171, "<|reserved_special_token_167|>": 128172, "<|reserved_special_token_168|>": 128173, "<|reserved_special_token_169|>": 128174, "<|reserved_special_token_170|>": 128175, "<|reserved_special_token_171|>": 128176, "<|reserved_special_token_172|>": 128177, "<|reserved_special_token_173|>": 128178, "<|reserved_special_token_174|>": 128179, "<|reserved_special_token_175|>": 128180, "<|reserved_special_token_176|>": 128181, "<|reserved_special_token_177|>": 128182, "<|reserved_special_token_178|>": 128183, "<|reserved_special_token_179|>": 128184, "<|reserved_special_token_180|>": 128185, "<|reserved_special_token_181|>": 128186, "<|reserved_special_token_182|>": 128187, "<|reserved_special_token_183|>": 128188, "<|reserved_special_token_184|>": 128189, "<|reserved_special_token_185|>": 128190, "<|reserved_special_token_186|>": 128191, "<|reserved_special_token_187|>": 128192, "<|reserved_special_token_188|>": 128193, "<|reserved_special_token_189|>": 128194, "<|reserved_special_token_190|>": 128195, "<|reserved_special_token_191|>": 128196, "<|reserved_special_token_192|>": 128197, "<|reserved_special_token_193|>": 128198, "<|reserved_special_token_194|>": 128199, "<|reserved_special_token_195|>": 128200, "<|reserved_special_token_196|>": 128201, "<|reserved_special_token_197|>": 128202, "<|reserved_special_token_198|>": 128203, "<|reserved_special_token_199|>": 128204, "<|reserved_special_token_200|>": 128205, "<|reserved_special_token_201|>": 128206, "<|reserved_special_token_202|>": 128207, "<|reserved_special_token_203|>": 128208, "<|reserved_special_token_204|>": 128209, "<|reserved_special_token_205|>": 128210, "<|reserved_special_token_206|>": 128211, "<|reserved_special_token_207|>": 128212, "<|reserved_special_token_208|>": 128213, "<|reserved_special_token_209|>": 128214, "<|reserved_special_token_210|>": 128215, "<|reserved_special_token_211|>": 128216, "<|reserved_special_token_212|>": 128217, "<|reserved_special_token_213|>": 128218, "<|reserved_special_token_214|>": 128219, "<|reserved_special_token_215|>": 128220, "<|reserved_special_token_216|>": 128221, "<|reserved_special_token_217|>": 128222, "<|reserved_special_token_218|>": 128223, "<|reserved_special_token_219|>": 128224, "<|reserved_special_token_220|>": 128225, "<|reserved_special_token_221|>": 128226, "<|reserved_special_token_222|>": 128227, "<|reserved_special_token_223|>": 128228, "<|reserved_special_token_224|>": 128229, "<|reserved_special_token_225|>": 128230, "<|reserved_special_token_226|>": 128231, "<|reserved_special_token_227|>": 128232, "<|reserved_special_token_228|>": 128233, "<|reserved_special_token_229|>": 128234, "<|reserved_special_token_230|>": 128235, "<|reserved_special_token_231|>": 128236, "<|reserved_special_token_232|>": 128237, "<|reserved_special_token_233|>": 128238, "<|reserved_special_token_234|>": 128239, "<|reserved_special_token_235|>": 128240, "<|reserved_special_token_236|>": 128241, "<|reserved_special_token_237|>": 128242, "<|reserved_special_token_238|>": 128243, "<|reserved_special_token_239|>": 128244, "<|reserved_special_token_240|>": 128245, "<|reserved_special_token_241|>": 128246, "<|reserved_special_token_242|>": 128247, "<|reserved_special_token_243|>": 128248, "<|reserved_special_token_244|>": 128249, "<|reserved_special_token_245|>": 128250, "<|reserved_special_token_246|>": 128251, "<|reserved_special_token_247|>": 128252, "<|reserved_special_token_248|>": 128253, "<|reserved_special_token_249|>": 128254, "<|reserved_special_token_250|>": 128255, "<|sound_start|>": 128256, "<|sound_end|>": 128257}