|
{ |
|
"add_bos_token": true, |
|
"add_eos_token": false, |
|
"added_tokens_decoder": { |
|
"0": { |
|
"content": "<pad>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"1": { |
|
"content": "<eos>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"2": { |
|
"content": "<bos>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"3": { |
|
"content": "<unk>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"4": { |
|
"content": "<mask>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"5": { |
|
"content": "<2mass>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"6": { |
|
"content": "[@BOS@]", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"7": { |
|
"content": "<|im_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"8": { |
|
"content": "<|im_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"9": { |
|
"content": "<unused2>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"10": { |
|
"content": "<unused3>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"11": { |
|
"content": "<unused4>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"12": { |
|
"content": "<unused5>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"13": { |
|
"content": "<unused6>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"14": { |
|
"content": "<unused7>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"15": { |
|
"content": "<unused8>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"16": { |
|
"content": "<unused9>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"17": { |
|
"content": "<unused10>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"18": { |
|
"content": "<unused11>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"19": { |
|
"content": "<unused12>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"20": { |
|
"content": "<unused13>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"21": { |
|
"content": "<unused14>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"22": { |
|
"content": "<unused15>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"23": { |
|
"content": "<unused16>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"24": { |
|
"content": "<unused17>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"25": { |
|
"content": "<unused18>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"26": { |
|
"content": "<unused19>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"27": { |
|
"content": "<unused20>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"28": { |
|
"content": "<unused21>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"29": { |
|
"content": "<unused22>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"30": { |
|
"content": "<unused23>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"31": { |
|
"content": "<unused24>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"32": { |
|
"content": "<unused25>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"33": { |
|
"content": "<unused26>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"34": { |
|
"content": "<unused27>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"35": { |
|
"content": "<unused28>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"36": { |
|
"content": "<unused29>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"37": { |
|
"content": "<unused30>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"38": { |
|
"content": "<unused31>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"39": { |
|
"content": "<unused32>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"40": { |
|
"content": "<unused33>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"41": { |
|
"content": "<unused34>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"42": { |
|
"content": "<unused35>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"43": { |
|
"content": "<unused36>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"44": { |
|
"content": "<unused37>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"45": { |
|
"content": "<unused38>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"46": { |
|
"content": "<unused39>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"47": { |
|
"content": "<unused40>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"48": { |
|
"content": "<unused41>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"49": { |
|
"content": "<unused42>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50": { |
|
"content": "<unused43>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"51": { |
|
"content": "<unused44>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"52": { |
|
"content": "<unused45>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"53": { |
|
"content": "<unused46>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"54": { |
|
"content": "<unused47>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"55": { |
|
"content": "<unused48>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"56": { |
|
"content": "<unused49>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"57": { |
|
"content": "<unused50>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"58": { |
|
"content": "<unused51>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"59": { |
|
"content": "<unused52>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"60": { |
|
"content": "<unused53>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"61": { |
|
"content": "<unused54>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"62": { |
|
"content": "<unused55>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"63": { |
|
"content": "<unused56>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"64": { |
|
"content": "<unused57>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"65": { |
|
"content": "<unused58>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"66": { |
|
"content": "<unused59>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"67": { |
|
"content": "<unused60>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"68": { |
|
"content": "<unused61>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"69": { |
|
"content": "<unused62>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"70": { |
|
"content": "<unused63>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"71": { |
|
"content": "<unused64>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"72": { |
|
"content": "<unused65>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"73": { |
|
"content": "<unused66>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"74": { |
|
"content": "<unused67>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"75": { |
|
"content": "<unused68>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"76": { |
|
"content": "<unused69>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"77": { |
|
"content": "<unused70>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"78": { |
|
"content": "<unused71>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"79": { |
|
"content": "<unused72>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"80": { |
|
"content": "<unused73>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"81": { |
|
"content": "<unused74>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"82": { |
|
"content": "<unused75>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"83": { |
|
"content": "<unused76>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"84": { |
|
"content": "<unused77>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"85": { |
|
"content": "<unused78>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"86": { |
|
"content": "<unused79>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"87": { |
|
"content": "<unused80>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"88": { |
|
"content": "<unused81>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"89": { |
|
"content": "<unused82>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"90": { |
|
"content": "<unused83>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"91": { |
|
"content": "<unused84>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"92": { |
|
"content": "<unused85>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"93": { |
|
"content": "<unused86>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"94": { |
|
"content": "<unused87>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"95": { |
|
"content": "<unused88>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"96": { |
|
"content": "<unused89>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"97": { |
|
"content": "<unused90>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"98": { |
|
"content": "<unused91>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"99": { |
|
"content": "<unused92>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"100": { |
|
"content": "<unused93>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"101": { |
|
"content": "<unused94>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"102": { |
|
"content": "<unused95>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"103": { |
|
"content": "<unused96>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"104": { |
|
"content": "<unused97>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"105": { |
|
"content": "<unused98>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"106": { |
|
"content": "<start_of_turn>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"107": { |
|
"content": "<end_of_turn>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"108": { |
|
"content": "\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"109": { |
|
"content": "\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"110": { |
|
"content": "\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"111": { |
|
"content": "\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"112": { |
|
"content": "\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"113": { |
|
"content": "\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"114": { |
|
"content": "\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"115": { |
|
"content": "\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"116": { |
|
"content": "\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"117": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"118": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"119": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"120": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"121": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"122": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"123": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"124": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"125": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"126": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"127": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"128": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"129": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"130": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"131": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"132": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"133": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"134": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"135": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"136": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"137": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"138": { |
|
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"139": { |
|
"content": "▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"140": { |
|
"content": "▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"141": { |
|
"content": "▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"142": { |
|
"content": "▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"143": { |
|
"content": "▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"144": { |
|
"content": "▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"145": { |
|
"content": "▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"146": { |
|
"content": "▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"147": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"148": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"149": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"150": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"152": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"153": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"154": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"155": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"156": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"157": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"158": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"159": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"160": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"161": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"162": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"163": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"164": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"165": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"166": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"167": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"168": { |
|
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"169": { |
|
"content": "<table>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"170": { |
|
"content": "<caption>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"171": { |
|
"content": "<thead>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"172": { |
|
"content": "<tbody>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"173": { |
|
"content": "<tfoot>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"174": { |
|
"content": "<tr>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"175": { |
|
"content": "<th>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"176": { |
|
"content": "<td>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"177": { |
|
"content": "</table>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"178": { |
|
"content": "</caption>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"179": { |
|
"content": "</thead>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"180": { |
|
"content": "</tbody>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"181": { |
|
"content": "</tfoot>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"182": { |
|
"content": "</tr>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"183": { |
|
"content": "</th>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"184": { |
|
"content": "</td>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"185": { |
|
"content": "<h1>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"186": { |
|
"content": "<h2>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"187": { |
|
"content": "<h3>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"188": { |
|
"content": "<h4>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"189": { |
|
"content": "<h5>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"190": { |
|
"content": "<h6>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"191": { |
|
"content": "<blockquote>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"192": { |
|
"content": "</h1>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"193": { |
|
"content": "</h2>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"194": { |
|
"content": "</h3>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"195": { |
|
"content": "</h4>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"196": { |
|
"content": "</h5>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"197": { |
|
"content": "</h6>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"198": { |
|
"content": "</blockquote>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"199": { |
|
"content": "<strong>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"200": { |
|
"content": "<em>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"201": { |
|
"content": "<b>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"202": { |
|
"content": "<i>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"203": { |
|
"content": "<u>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"204": { |
|
"content": "<s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"205": { |
|
"content": "<sub>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"206": { |
|
"content": "<sup>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"207": { |
|
"content": "<code>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"208": { |
|
"content": "</strong>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"209": { |
|
"content": "</em>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"210": { |
|
"content": "</b>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"211": { |
|
"content": "</i>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"212": { |
|
"content": "</u>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"213": { |
|
"content": "</s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"214": { |
|
"content": "</sub>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"215": { |
|
"content": "</sup>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"216": { |
|
"content": "</code>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"<start_of_turn>", |
|
"<end_of_turn>" |
|
], |
|
"bos_token": "<bos>", |
|
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", |
|
"clean_up_tokenization_spaces": false, |
|
"eos_token": "<|im_end|>", |
|
"model_max_length": 1000000000000000019884624838656, |
|
"pad_token": "<pad>", |
|
"sp_model_kwargs": {}, |
|
"spaces_between_special_tokens": false, |
|
"tokenizer_class": "GemmaTokenizer", |
|
"unk_token": "<unk>", |
|
"use_default_system_prompt": false |
|
} |
|
|