Spaces:
Running
Running
File size: 1,597 Bytes
9495a4f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
import json
import os
from transformers import LlamaTokenizer
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
TOKENIZER_DIR = os.path.join(CURRENT_DIR, "tokenizer")
tokenizer = LlamaTokenizer.from_pretrained(TOKENIZER_DIR)
tokens = [ 1, 29961, 25580, 29962, 3532, 14816, 29903, 6778, 13, 3492,
526, 263, 8444, 29892, 3390, 1319, 322, 15993, 20255, 29889,
29849, 1234, 408, 1371, 3730, 408, 1950, 29892, 1550, 1641,
9109, 29889, 29871, 3575, 6089, 881, 451, 3160, 738, 10311,
1319, 29892, 443, 621, 936, 29892, 11021, 391, 29892, 7916,
391, 29892, 304, 27375, 29892, 18215, 29892, 470, 27302, 2793,
29889, 3529, 9801, 393, 596, 20890, 526, 5374, 635, 443,
5365, 1463, 322, 6374, 297, 5469, 29889, 13, 13, 3644,
263, 1139, 947, 451, 1207, 738, 4060, 29892, 470, 338,
451, 2114, 1474, 16165, 261, 296, 29892, 5649, 2020, 2012,
310, 22862, 1554, 451, 1959, 29889, 960, 366, 1016, 29915,
29873, 1073, 278, 1234, 304, 263, 1139, 29892, 3113, 1016,
29915, 29873, 6232, 2089, 2472, 29889, 13, 29966, 829, 14816,
29903, 6778, 13, 13, 15970, 526, 366, 518, 29914, 25580,
29962]
text = tokenizer.decode(tokens)
print(text)
for token_id in tokens:
print(json.dumps({"token_id": token_id, "decode_str": tokenizer.decode([token_id]), "token": tokenizer.convert_ids_to_tokens([token_id][0])}, ensure_ascii=False)) |