jonathanjordan21 commited on
Commit
302bcc8
1 Parent(s): f1b5654

Update constants/models.py

Browse files
Files changed (1) hide show
  1. constants/models.py +3 -0
constants/models.py CHANGED
@@ -34,6 +34,9 @@ STOP_SEQUENCES_MAP = {
34
  "command-r-plus": "<|END_OF_TURN_TOKEN|>",
35
  # https://huggingface.co/Qwen/Qwen2-72B-Instruct/blob/main/tokenizer_config.json#L30
36
  "Qwen2-72B": "<|im_end|>",
 
 
 
37
  }
38
 
39
  TOKEN_LIMIT_MAP = {
 
34
  "command-r-plus": "<|END_OF_TURN_TOKEN|>",
35
  # https://huggingface.co/Qwen/Qwen2-72B-Instruct/blob/main/tokenizer_config.json#L30
36
  "Qwen2-72B": "<|im_end|>",
37
+ "Qwen2-7B-Instruct": "<im_end|>",
38
+
39
+ "llama3" : "<|eot_id|>"
40
  }
41
 
42
  TOKEN_LIMIT_MAP = {