Upload 9 files

openvino_detokenizer.xml

@@ -93,7 +93,7 @@
 	</edges>
 	<rt_info>
 		<bos_token_id value="1" />
-		<chat_template value="{% for message in messages %}{% if message['role'] == 'system' %}{{'&lt;|system|>&#10;' + message['content'] + '&lt;|end|>&#10;'}}{% elif message['role'] == 'user' %}{{'&lt;|user|>&#10;' + message['content'] + '&lt;|end|>&#10;'}}{% elif message['role'] == 'assistant' %}{{'&lt;|assistant|>&#10;' + message['content'] + '&lt;|end|>&#10;'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '&lt;|assistant|>&#10;' }}{% else %}{{ eos_token }}{% endif %}" />
+		<chat_template value="{% for message in messages %}{% if (message['role'] == 'user') %}{{'&lt;|user|>' + '&#10;' + message['content'] + '&lt;|end|>' + '&#10;' + '&lt;|assistant|>' + '&#10;'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '&lt;|end|>' + '&#10;'}}{% endif %}{% endfor %}" />
 		<eos_token_id value="32000" />
 		<original_tokenizer_class value="&lt;class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>" />
 		<pad_token_id value="32000" />

openvino_tokenizer.xml

@@ -429,7 +429,7 @@
 	</edges>
 	<rt_info>
 		<bos_token_id value="1" />
-		<chat_template value="{% for message in messages %}{% if message['role'] == 'system' %}{{'&lt;|system|>&#10;' + message['content'] + '&lt;|end|>&#10;'}}{% elif message['role'] == 'user' %}{{'&lt;|user|>&#10;' + message['content'] + '&lt;|end|>&#10;'}}{% elif message['role'] == 'assistant' %}{{'&lt;|assistant|>&#10;' + message['content'] + '&lt;|end|>&#10;'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '&lt;|assistant|>&#10;' }}{% else %}{{ eos_token }}{% endif %}" />
+		<chat_template value="{% for message in messages %}{% if (message['role'] == 'user') %}{{'&lt;|user|>' + '&#10;' + message['content'] + '&lt;|end|>' + '&#10;' + '&lt;|assistant|>' + '&#10;'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '&lt;|end|>' + '&#10;'}}{% endif %}{% endfor %}" />
 		<eos_token_id value="32000" />
 		<original_tokenizer_class value="&lt;class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>" />
 		<pad_token_id value="32000" />

tokenizer_config.json

@@ -117,7 +117,7 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "chat_template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,