Add "chat_template" to tokenizer_config.json (#1)
Browse files- Add "chat_template" to tokenizer_config.json (2de2f1206c4927755e9471b8ce1e9b6c14f5d469)
- Add assistant role (28979cfbb62e70f1e7d811ff6d5c2d0c1e6f7fe3)
- Update tokenizer_config.json (e6f096488fbc19fd5e2421cf730c5824d5c0a8ed)
- Add assistant content to prompt (f8e1bb1560d7dfc88e2b713a33ce11045d1deaef)
Co-authored-by: Shunta Saito <[email protected]>
- tokenizer_config.json +1 -0
tokenizer_config.json
CHANGED
@@ -75,6 +75,7 @@
|
|
75 |
]
|
76 |
},
|
77 |
"bos_token": "<s>",
|
|
|
78 |
"clean_up_tokenization_spaces": false,
|
79 |
"cls_token": "<cls>",
|
80 |
"eos_token": "</s>",
|
|
|
75 |
]
|
76 |
},
|
77 |
"bos_token": "<s>",
|
78 |
+
"chat_template": "{{ '以下はタスクを説明する指示で、文脈を説明した入力とペアになっています。要求を適切に補完するよう応答を書いてください。\n\n' }}{% for message in messages %}{% if message['role'] == 'system' %}{{ '### 指示:\n' + message['content'].strip() + '\n\n' }}{% elif message['role'] == 'user' %}{{ '### 入力:\n' + message['content'].strip() + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### 応答:\n' + message['content'].strip() + '\n\n' }}{% endif %}{% endfor %}{{ '### 応答:\n' }}",
|
79 |
"clean_up_tokenization_spaces": false,
|
80 |
"cls_token": "<cls>",
|
81 |
"eos_token": "</s>",
|