misdelivery
commited on
Commit
•
e81977e
1
Parent(s):
fe95839
Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -0
tokenizer_config.json
CHANGED
@@ -75,6 +75,8 @@
|
|
75 |
"extra_ids": 0,
|
76 |
"mask_token": "<MASK>",
|
77 |
"model_max_length": 1000000000000000019884624838656,
|
|
|
|
|
78 |
"pad_token": "<pad>",
|
79 |
"sep_token": "<SEP>",
|
80 |
"sp_model_kwargs": {},
|
|
|
75 |
"extra_ids": 0,
|
76 |
"mask_token": "<MASK>",
|
77 |
"model_max_length": 1000000000000000019884624838656,
|
78 |
+
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '\n\n### 指示:\n' + message['content'] }}{% elif message['role'] == 'system' %}{{ '以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。' }}{% elif message['role'] == 'assistant' %}{{ '\n\n### 応答:\n' + message['content'] + eos_token }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '\\n\\n### 応答:\\n' }}{% endif %}{% endfor %}",
|
79 |
+
"clean_up_tokenization_spaces": false,
|
80 |
"pad_token": "<pad>",
|
81 |
"sep_token": "<SEP>",
|
82 |
"sp_model_kwargs": {},
|