ZhangYuanhan commited on
Commit
5b49db1
1 Parent(s): 24d75c5

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +10 -1
tokenizer_config.json CHANGED
@@ -24,6 +24,14 @@
24
  "rstrip": false,
25
  "single_word": false,
26
  "special": true
 
 
 
 
 
 
 
 
27
  }
28
  },
29
  "additional_special_tokens": [
@@ -38,7 +46,8 @@
38
  "model_max_length": 32768,
39
  "pad_token": "<|endoftext|>",
40
  "padding_side": "right",
 
41
  "split_special_tokens": false,
42
  "tokenizer_class": "Qwen2Tokenizer",
43
  "unk_token": null
44
- }
 
24
  "rstrip": false,
25
  "single_word": false,
26
  "special": true
27
+ },
28
+ "151646": {
29
+ "content": "<image>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
  }
36
  },
37
  "additional_special_tokens": [
 
46
  "model_max_length": 32768,
47
  "pad_token": "<|endoftext|>",
48
  "padding_side": "right",
49
+ "processor_class": "LlavaProcessor",
50
  "split_special_tokens": false,
51
  "tokenizer_class": "Qwen2Tokenizer",
52
  "unk_token": null
53
+ }