DeepMount00 commited on
Commit
e9d603a
1 Parent(s): 8e64485

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +1 -0
  2. tokenizer_config.json +2 -1
special_tokens_map.json CHANGED
@@ -13,6 +13,7 @@
13
  "rstrip": false,
14
  "single_word": false
15
  },
 
16
  "unk_token": {
17
  "content": "<unk>",
18
  "lstrip": false,
 
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": "</s>",
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -27,11 +27,12 @@
27
  },
28
  "additional_special_tokens": [],
29
  "bos_token": "<s>",
 
30
  "clean_up_tokenization_spaces": false,
31
  "eos_token": "</s>",
32
  "legacy": true,
33
  "model_max_length": 1000000000000000019884624838656,
34
- "pad_token": null,
35
  "sp_model_kwargs": {},
36
  "spaces_between_special_tokens": false,
37
  "tokenizer_class": "LlamaTokenizer",
 
27
  },
28
  "additional_special_tokens": [],
29
  "bos_token": "<s>",
30
+ "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
31
  "clean_up_tokenization_spaces": false,
32
  "eos_token": "</s>",
33
  "legacy": true,
34
  "model_max_length": 1000000000000000019884624838656,
35
+ "pad_token": "</s>",
36
  "sp_model_kwargs": {},
37
  "spaces_between_special_tokens": false,
38
  "tokenizer_class": "LlamaTokenizer",