dialofred / added_tokens.json
TeraSpace's picture
Upload tokenizer
ebd2a57
raw
history blame
2.74 kB
{
"<LM>": 50357,
"<SC1>": 50358,
"<SC2>": 50359,
"<SC3>": 50360,
"<SC4>": 50361,
"<SC5>": 50362,
"<SC6>": 50363,
"<extra_id_0>": 50257,
"<extra_id_10>": 50267,
"<extra_id_11>": 50268,
"<extra_id_12>": 50269,
"<extra_id_13>": 50270,
"<extra_id_14>": 50271,
"<extra_id_15>": 50272,
"<extra_id_16>": 50273,
"<extra_id_17>": 50274,
"<extra_id_18>": 50275,
"<extra_id_19>": 50276,
"<extra_id_1>": 50258,
"<extra_id_20>": 50277,
"<extra_id_21>": 50278,
"<extra_id_22>": 50279,
"<extra_id_23>": 50280,
"<extra_id_24>": 50281,
"<extra_id_25>": 50282,
"<extra_id_26>": 50283,
"<extra_id_27>": 50284,
"<extra_id_28>": 50285,
"<extra_id_29>": 50286,
"<extra_id_2>": 50259,
"<extra_id_30>": 50287,
"<extra_id_31>": 50288,
"<extra_id_32>": 50289,
"<extra_id_33>": 50290,
"<extra_id_34>": 50291,
"<extra_id_35>": 50292,
"<extra_id_36>": 50293,
"<extra_id_37>": 50294,
"<extra_id_38>": 50295,
"<extra_id_39>": 50296,
"<extra_id_3>": 50260,
"<extra_id_40>": 50297,
"<extra_id_41>": 50298,
"<extra_id_42>": 50299,
"<extra_id_43>": 50300,
"<extra_id_44>": 50301,
"<extra_id_45>": 50302,
"<extra_id_46>": 50303,
"<extra_id_47>": 50304,
"<extra_id_48>": 50305,
"<extra_id_49>": 50306,
"<extra_id_4>": 50261,
"<extra_id_50>": 50307,
"<extra_id_51>": 50308,
"<extra_id_52>": 50309,
"<extra_id_53>": 50310,
"<extra_id_54>": 50311,
"<extra_id_55>": 50312,
"<extra_id_56>": 50313,
"<extra_id_57>": 50314,
"<extra_id_58>": 50315,
"<extra_id_59>": 50316,
"<extra_id_5>": 50262,
"<extra_id_60>": 50317,
"<extra_id_61>": 50318,
"<extra_id_62>": 50319,
"<extra_id_63>": 50320,
"<extra_id_64>": 50321,
"<extra_id_65>": 50322,
"<extra_id_66>": 50323,
"<extra_id_67>": 50324,
"<extra_id_68>": 50325,
"<extra_id_69>": 50326,
"<extra_id_6>": 50263,
"<extra_id_70>": 50327,
"<extra_id_71>": 50328,
"<extra_id_72>": 50329,
"<extra_id_73>": 50330,
"<extra_id_74>": 50331,
"<extra_id_75>": 50332,
"<extra_id_76>": 50333,
"<extra_id_77>": 50334,
"<extra_id_78>": 50335,
"<extra_id_79>": 50336,
"<extra_id_7>": 50264,
"<extra_id_80>": 50337,
"<extra_id_81>": 50338,
"<extra_id_82>": 50339,
"<extra_id_83>": 50340,
"<extra_id_84>": 50341,
"<extra_id_85>": 50342,
"<extra_id_86>": 50343,
"<extra_id_87>": 50344,
"<extra_id_88>": 50345,
"<extra_id_89>": 50346,
"<extra_id_8>": 50265,
"<extra_id_90>": 50347,
"<extra_id_91>": 50348,
"<extra_id_92>": 50349,
"<extra_id_93>": 50350,
"<extra_id_94>": 50351,
"<extra_id_95>": 50352,
"<extra_id_96>": 50353,
"<extra_id_97>": 50354,
"<extra_id_98>": 50355,
"<extra_id_99>": 50356,
"<extra_id_9>": 50266,
"<|endoftext|>": 50364
}