finalf0 commited on
Commit
fda92ea
1 Parent(s): 6e50765
config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "openbmb/MiniCPM-V-2",
3
+ "architectures": [
4
+ "MiniCPMV"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "auto_map": {
9
+ "AutoConfig": "configuration_minicpm.MiniCPMVConfig",
10
+ "AutoModel": "modeling_minicpmv.MiniCPMV",
11
+ "AutoModelForCausalLM": "modeling_minicpmv.MiniCPMV"
12
+ },
13
+ "bos_token_id": 1,
14
+ "dim_model_base": 256,
15
+ "drop_vision_last_layer": true,
16
+ "eos_token_id": 2,
17
+ "hidden_act": "silu",
18
+ "hidden_size": 2304,
19
+ "image_size": 448,
20
+ "initializer_range": 0.1,
21
+ "intermediate_size": 5760,
22
+ "max_position_embeddings": 4096,
23
+ "max_slice_nums": 9,
24
+ "mm_use_im_start_end": true,
25
+ "model_type": "minicpmv",
26
+ "num_attention_heads": 36,
27
+ "num_hidden_layers": 40,
28
+ "num_key_value_heads": 36,
29
+ "patch_size": 14,
30
+ "pretraining_tp": 1,
31
+ "query_num": 64,
32
+ "rms_norm_eps": 1e-05,
33
+ "rope_scaling": null,
34
+ "rope_theta": 10000.0,
35
+ "scale_depth": 1.4,
36
+ "scale_emb": 12,
37
+ "scale_resolution": 448,
38
+ "slice_mode": true,
39
+ "tie_word_embeddings": false,
40
+ "torch_dtype": "bfloat16",
41
+ "transformers_version": "4.36.0",
42
+ "use_cache": true,
43
+ "vision_encoder": "vit_so400m_patch14_siglip_384.webli",
44
+ "vocab_size": 122753
45
+ }
configuration_minicpm.py ADDED
@@ -0,0 +1,222 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+ # and OPT implementations in this library. It has been modified from its
6
+ # original forms to accommodate minor architectural differences compared
7
+ # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+ """ MiniCPM model configuration"""
21
+
22
+ from transformers.configuration_utils import PretrainedConfig
23
+ from transformers.utils import logging
24
+
25
+ logger = logging.get_logger(__name__)
26
+
27
+ MINICPM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
28
+
29
+
30
+ class MiniCPMConfig(PretrainedConfig):
31
+ r"""
32
+ This is the configuration class to store the configuration of a [`MiniCPMModel`]. It is used to instantiate an MiniCPM
33
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
34
+ defaults will yield a similar configuration to that of the MiniCPM-7B.
35
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
36
+ documentation from [`PretrainedConfig`] for more information.
37
+ Args:
38
+ vocab_size (`int`, *optional*, defaults to 32000):
39
+ Vocabulary size of the MiniCPM model. Defines the number of different tokens that can be represented by the
40
+ `inputs_ids` passed when calling [`MiniCPMModel`]
41
+ hidden_size (`int`, *optional*, defaults to 4096):
42
+ Dimension of the hidden representations.
43
+ intermediate_size (`int`, *optional*, defaults to 11008):
44
+ Dimension of the MLP representations.
45
+ num_hidden_layers (`int`, *optional*, defaults to 32):
46
+ Number of hidden layers in the Transformer decoder.
47
+ num_attention_heads (`int`, *optional*, defaults to 32):
48
+ Number of attention heads for each attention layer in the Transformer decoder.
49
+ num_key_value_heads (`int`, *optional*):
50
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
51
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
52
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
53
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
54
+ by meanpooling all the original heads within that group. For more details checkout [this
55
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
56
+ `num_attention_heads`.
57
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
58
+ The non-linear activation function (function or string) in the decoder.
59
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
60
+ The maximum sequence length that this model might ever be used with. MiniCPM 1 supports up to 2048 tokens,
61
+ MiniCPM 2 up to 4096, CodeMiniCPM up to 16384.
62
+ initializer_range (`float`, *optional*, defaults to 0.02):
63
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
64
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
65
+ The epsilon used by the rms normalization layers.
66
+ use_cache (`bool`, *optional*, defaults to `True`):
67
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
68
+ relevant if `config.is_decoder=True`.
69
+ pad_token_id (`int`, *optional*):
70
+ Padding token id.
71
+ bos_token_id (`int`, *optional*, defaults to 1):
72
+ Beginning of stream token id.
73
+ eos_token_id (`int`, *optional*, defaults to 2):
74
+ End of stream token id.
75
+ pretraining_tp (`int`, *optional*, defaults to 1):
76
+ Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
77
+ document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
78
+ necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
79
+ issue](https://github.com/pytorch/pytorch/issues/76232).
80
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
81
+ Whether to tie weight embeddings
82
+ rope_theta (`float`, *optional*, defaults to 10000.0):
83
+ The base period of the RoPE embeddings.
84
+ rope_scaling (`Dict`, *optional*):
85
+ Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
86
+ strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
87
+ `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
88
+ `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
89
+ these scaling strategies behave:
90
+ https://www.reddit.com/r/LocalMiniCPM/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
91
+ experimental feature, subject to breaking API changes in future versions.
92
+ attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
93
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
94
+ attention_dropout (`float`, *optional*, defaults to 0.0):
95
+ The dropout ratio for the attention probabilities.
96
+ ```python
97
+ >>> from transformers import MiniCPMModel, MiniCPMConfig
98
+ >>> # Initializing a MiniCPM minicpm-7b style configuration
99
+ >>> configuration = MiniCPMConfig()
100
+ >>> # Initializing a model from the minicpm-7b style configuration
101
+ >>> model = MiniCPMModel(configuration)
102
+ >>> # Accessing the model configuration
103
+ >>> configuration = model.config
104
+ ```"""
105
+
106
+ model_type = "minicpm"
107
+ keys_to_ignore_at_inference = ["past_key_values"]
108
+
109
+ def __init__(
110
+ self,
111
+ vocab_size=32000,
112
+ hidden_size=4096,
113
+ intermediate_size=11008,
114
+ num_hidden_layers=32,
115
+ num_attention_heads=32,
116
+ num_key_value_heads=None,
117
+ hidden_act="silu",
118
+ max_position_embeddings=2048,
119
+ initializer_range=0.02,
120
+ rms_norm_eps=1e-6,
121
+ use_cache=True,
122
+ pad_token_id=None,
123
+ bos_token_id=1,
124
+ eos_token_id=2,
125
+ pretraining_tp=1,
126
+ tie_word_embeddings=False,
127
+ rope_theta=10000.0,
128
+ rope_scaling=None,
129
+ attention_bias=False,
130
+ attention_dropout=0.0,
131
+ scale_emb=1,
132
+ dim_model_base=1,
133
+ scale_depth=1,
134
+ **kwargs,
135
+ ):
136
+ self.vocab_size = vocab_size
137
+ self.max_position_embeddings = max_position_embeddings
138
+ self.hidden_size = hidden_size
139
+ self.intermediate_size = intermediate_size
140
+ self.num_hidden_layers = num_hidden_layers
141
+ self.num_attention_heads = num_attention_heads
142
+
143
+ # for backward compatibility
144
+ if num_key_value_heads is None:
145
+ num_key_value_heads = num_attention_heads
146
+
147
+ self.num_key_value_heads = num_key_value_heads
148
+ self.hidden_act = hidden_act
149
+ self.initializer_range = initializer_range
150
+ self.rms_norm_eps = rms_norm_eps
151
+ self.pretraining_tp = pretraining_tp
152
+ self.use_cache = use_cache
153
+ self.rope_theta = rope_theta
154
+ self.rope_scaling = rope_scaling
155
+ self._rope_scaling_validation()
156
+ self.attention_bias = attention_bias
157
+ self.attention_dropout = attention_dropout
158
+ self.scale_emb = scale_emb
159
+ self.dim_model_base = dim_model_base
160
+ self.scale_depth = scale_depth
161
+
162
+ super().__init__(
163
+ pad_token_id=pad_token_id,
164
+ bos_token_id=bos_token_id,
165
+ eos_token_id=eos_token_id,
166
+ tie_word_embeddings=tie_word_embeddings,
167
+ **kwargs,
168
+ )
169
+
170
+ def _rope_scaling_validation(self):
171
+ """
172
+ Validate the `rope_scaling` configuration.
173
+ """
174
+ if self.rope_scaling is None:
175
+ return
176
+
177
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
178
+ raise ValueError(
179
+ "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
180
+ f"got {self.rope_scaling}"
181
+ )
182
+ rope_scaling_type = self.rope_scaling.get("type", None)
183
+ rope_scaling_factor = self.rope_scaling.get("factor", None)
184
+ if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
185
+ raise ValueError(
186
+ f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
187
+ )
188
+ if (
189
+ rope_scaling_factor is None
190
+ or not isinstance(rope_scaling_factor, float)
191
+ or rope_scaling_factor <= 1.0
192
+ ):
193
+ raise ValueError(
194
+ f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}"
195
+ )
196
+
197
+
198
+ class MiniCPMVConfig(MiniCPMConfig):
199
+ model_type = "minicpmv"
200
+ keys_to_ignore_at_inference = ["past_key_values"]
201
+
202
+ def __init__(
203
+ self,
204
+ vision_encoder="vit_so400m_patch14_siglip_384.webli",
205
+ query_num=64,
206
+ image_size=448,
207
+ drop_vision_last_layer=True,
208
+ slice_mode=True,
209
+ patch_size=14,
210
+ max_slice_nums=9,
211
+ scale_resolution=448,
212
+ **kwargs,
213
+ ):
214
+ self.vision_encoder = vision_encoder
215
+ self.query_num = query_num
216
+ self.image_size = image_size
217
+ self.drop_vision_last_layer = drop_vision_last_layer
218
+ self.slice_mode = slice_mode
219
+ self.patch_size = patch_size
220
+ self.max_slice_nums = max_slice_nums
221
+ self.scale_resolution = scale_resolution
222
+ super().__init__(**kwargs)
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.36.0"
6
+ }
model.safetensors.index.json ADDED
@@ -0,0 +1,701 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 6869931584
4
+ },
5
+ "weight_map": {
6
+ "llm.lm_head.weight": "model-00002-of-00002.safetensors",
7
+ "llm.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
8
+ "llm.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
9
+ "llm.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
10
+ "llm.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
11
+ "llm.model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
12
+ "llm.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
13
+ "llm.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
14
+ "llm.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
15
+ "llm.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
16
+ "llm.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
17
+ "llm.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
18
+ "llm.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
19
+ "llm.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
20
+ "llm.model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
21
+ "llm.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
22
+ "llm.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
23
+ "llm.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
24
+ "llm.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
25
+ "llm.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
26
+ "llm.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
27
+ "llm.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
28
+ "llm.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
29
+ "llm.model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
30
+ "llm.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
31
+ "llm.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
32
+ "llm.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
33
+ "llm.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
34
+ "llm.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
35
+ "llm.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
36
+ "llm.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
37
+ "llm.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
38
+ "llm.model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
39
+ "llm.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
40
+ "llm.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
41
+ "llm.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
42
+ "llm.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
43
+ "llm.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
44
+ "llm.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
45
+ "llm.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
46
+ "llm.model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
47
+ "llm.model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
48
+ "llm.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
49
+ "llm.model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
50
+ "llm.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
51
+ "llm.model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
52
+ "llm.model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
53
+ "llm.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
54
+ "llm.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
55
+ "llm.model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
56
+ "llm.model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
57
+ "llm.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
58
+ "llm.model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
59
+ "llm.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
60
+ "llm.model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
61
+ "llm.model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
62
+ "llm.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
63
+ "llm.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
64
+ "llm.model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
65
+ "llm.model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
66
+ "llm.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
67
+ "llm.model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
68
+ "llm.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
69
+ "llm.model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
70
+ "llm.model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
71
+ "llm.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
72
+ "llm.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
73
+ "llm.model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
74
+ "llm.model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
75
+ "llm.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
76
+ "llm.model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
77
+ "llm.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
78
+ "llm.model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
79
+ "llm.model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
80
+ "llm.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
81
+ "llm.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
82
+ "llm.model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
83
+ "llm.model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
84
+ "llm.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
85
+ "llm.model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
86
+ "llm.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
87
+ "llm.model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
88
+ "llm.model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
89
+ "llm.model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
90
+ "llm.model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
91
+ "llm.model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
92
+ "llm.model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
93
+ "llm.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
94
+ "llm.model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
95
+ "llm.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
96
+ "llm.model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
97
+ "llm.model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
98
+ "llm.model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
99
+ "llm.model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
100
+ "llm.model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
101
+ "llm.model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
102
+ "llm.model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
103
+ "llm.model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
104
+ "llm.model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
105
+ "llm.model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
106
+ "llm.model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
107
+ "llm.model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
108
+ "llm.model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
109
+ "llm.model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
110
+ "llm.model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
111
+ "llm.model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
112
+ "llm.model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
113
+ "llm.model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
114
+ "llm.model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
115
+ "llm.model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
116
+ "llm.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
117
+ "llm.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
118
+ "llm.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
119
+ "llm.model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
120
+ "llm.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
121
+ "llm.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
122
+ "llm.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
123
+ "llm.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
124
+ "llm.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
125
+ "llm.model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
126
+ "llm.model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
127
+ "llm.model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
128
+ "llm.model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
129
+ "llm.model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
130
+ "llm.model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
131
+ "llm.model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
132
+ "llm.model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
133
+ "llm.model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
134
+ "llm.model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
135
+ "llm.model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
136
+ "llm.model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
137
+ "llm.model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
138
+ "llm.model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
139
+ "llm.model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
140
+ "llm.model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
141
+ "llm.model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
142
+ "llm.model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
143
+ "llm.model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
144
+ "llm.model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
145
+ "llm.model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
146
+ "llm.model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
147
+ "llm.model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
148
+ "llm.model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
149
+ "llm.model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
150
+ "llm.model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
151
+ "llm.model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
152
+ "llm.model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
153
+ "llm.model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
154
+ "llm.model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
155
+ "llm.model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
156
+ "llm.model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
157
+ "llm.model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
158
+ "llm.model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
159
+ "llm.model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
160
+ "llm.model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
161
+ "llm.model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
162
+ "llm.model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
163
+ "llm.model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
164
+ "llm.model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
165
+ "llm.model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
166
+ "llm.model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
167
+ "llm.model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
168
+ "llm.model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
169
+ "llm.model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
170
+ "llm.model.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
171
+ "llm.model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
172
+ "llm.model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
173
+ "llm.model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
174
+ "llm.model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
175
+ "llm.model.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
176
+ "llm.model.layers.25.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
177
+ "llm.model.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
178
+ "llm.model.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
179
+ "llm.model.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
180
+ "llm.model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
181
+ "llm.model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
182
+ "llm.model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
183
+ "llm.model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
184
+ "llm.model.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
185
+ "llm.model.layers.26.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
186
+ "llm.model.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
187
+ "llm.model.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
188
+ "llm.model.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
189
+ "llm.model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
190
+ "llm.model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
191
+ "llm.model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
192
+ "llm.model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
193
+ "llm.model.layers.27.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
194
+ "llm.model.layers.27.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
195
+ "llm.model.layers.27.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
196
+ "llm.model.layers.27.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
197
+ "llm.model.layers.28.input_layernorm.weight": "model-00001-of-00002.safetensors",
198
+ "llm.model.layers.28.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
199
+ "llm.model.layers.28.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
200
+ "llm.model.layers.28.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
201
+ "llm.model.layers.28.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
202
+ "llm.model.layers.28.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
203
+ "llm.model.layers.28.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
204
+ "llm.model.layers.28.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
205
+ "llm.model.layers.28.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
206
+ "llm.model.layers.29.input_layernorm.weight": "model-00001-of-00002.safetensors",
207
+ "llm.model.layers.29.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
208
+ "llm.model.layers.29.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
209
+ "llm.model.layers.29.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
210
+ "llm.model.layers.29.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
211
+ "llm.model.layers.29.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
212
+ "llm.model.layers.29.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
213
+ "llm.model.layers.29.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
214
+ "llm.model.layers.29.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
215
+ "llm.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
216
+ "llm.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
217
+ "llm.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
218
+ "llm.model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
219
+ "llm.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
220
+ "llm.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
221
+ "llm.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
222
+ "llm.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
223
+ "llm.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
224
+ "llm.model.layers.30.input_layernorm.weight": "model-00001-of-00002.safetensors",
225
+ "llm.model.layers.30.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
226
+ "llm.model.layers.30.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
227
+ "llm.model.layers.30.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
228
+ "llm.model.layers.30.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
229
+ "llm.model.layers.30.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
230
+ "llm.model.layers.30.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
231
+ "llm.model.layers.30.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
232
+ "llm.model.layers.30.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
233
+ "llm.model.layers.31.input_layernorm.weight": "model-00001-of-00002.safetensors",
234
+ "llm.model.layers.31.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
235
+ "llm.model.layers.31.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
236
+ "llm.model.layers.31.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
237
+ "llm.model.layers.31.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
238
+ "llm.model.layers.31.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
239
+ "llm.model.layers.31.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
240
+ "llm.model.layers.31.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
241
+ "llm.model.layers.31.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
242
+ "llm.model.layers.32.input_layernorm.weight": "model-00001-of-00002.safetensors",
243
+ "llm.model.layers.32.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
244
+ "llm.model.layers.32.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
245
+ "llm.model.layers.32.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
246
+ "llm.model.layers.32.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
247
+ "llm.model.layers.32.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
248
+ "llm.model.layers.32.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
249
+ "llm.model.layers.32.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
250
+ "llm.model.layers.32.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
251
+ "llm.model.layers.33.input_layernorm.weight": "model-00001-of-00002.safetensors",
252
+ "llm.model.layers.33.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
253
+ "llm.model.layers.33.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
254
+ "llm.model.layers.33.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
255
+ "llm.model.layers.33.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
256
+ "llm.model.layers.33.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
257
+ "llm.model.layers.33.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
258
+ "llm.model.layers.33.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
259
+ "llm.model.layers.33.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
260
+ "llm.model.layers.34.input_layernorm.weight": "model-00001-of-00002.safetensors",
261
+ "llm.model.layers.34.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
262
+ "llm.model.layers.34.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
263
+ "llm.model.layers.34.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
264
+ "llm.model.layers.34.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
265
+ "llm.model.layers.34.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
266
+ "llm.model.layers.34.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
267
+ "llm.model.layers.34.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
268
+ "llm.model.layers.34.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
269
+ "llm.model.layers.35.input_layernorm.weight": "model-00001-of-00002.safetensors",
270
+ "llm.model.layers.35.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
271
+ "llm.model.layers.35.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
272
+ "llm.model.layers.35.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
273
+ "llm.model.layers.35.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
274
+ "llm.model.layers.35.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
275
+ "llm.model.layers.35.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
276
+ "llm.model.layers.35.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
277
+ "llm.model.layers.35.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
278
+ "llm.model.layers.36.input_layernorm.weight": "model-00002-of-00002.safetensors",
279
+ "llm.model.layers.36.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
280
+ "llm.model.layers.36.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
281
+ "llm.model.layers.36.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
282
+ "llm.model.layers.36.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
283
+ "llm.model.layers.36.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
284
+ "llm.model.layers.36.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
285
+ "llm.model.layers.36.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
286
+ "llm.model.layers.36.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
287
+ "llm.model.layers.37.input_layernorm.weight": "model-00002-of-00002.safetensors",
288
+ "llm.model.layers.37.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
289
+ "llm.model.layers.37.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
290
+ "llm.model.layers.37.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
291
+ "llm.model.layers.37.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
292
+ "llm.model.layers.37.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
293
+ "llm.model.layers.37.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
294
+ "llm.model.layers.37.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
295
+ "llm.model.layers.37.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
296
+ "llm.model.layers.38.input_layernorm.weight": "model-00002-of-00002.safetensors",
297
+ "llm.model.layers.38.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
298
+ "llm.model.layers.38.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
299
+ "llm.model.layers.38.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
300
+ "llm.model.layers.38.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
301
+ "llm.model.layers.38.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
302
+ "llm.model.layers.38.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
303
+ "llm.model.layers.38.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
304
+ "llm.model.layers.38.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
305
+ "llm.model.layers.39.input_layernorm.weight": "model-00002-of-00002.safetensors",
306
+ "llm.model.layers.39.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
307
+ "llm.model.layers.39.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
308
+ "llm.model.layers.39.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
309
+ "llm.model.layers.39.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
310
+ "llm.model.layers.39.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
311
+ "llm.model.layers.39.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
312
+ "llm.model.layers.39.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
313
+ "llm.model.layers.39.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
314
+ "llm.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
315
+ "llm.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
316
+ "llm.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
317
+ "llm.model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
318
+ "llm.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
319
+ "llm.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
320
+ "llm.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
321
+ "llm.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
322
+ "llm.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
323
+ "llm.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
324
+ "llm.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
325
+ "llm.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
326
+ "llm.model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
327
+ "llm.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
328
+ "llm.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
329
+ "llm.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
330
+ "llm.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
331
+ "llm.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
332
+ "llm.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
333
+ "llm.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
334
+ "llm.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
335
+ "llm.model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
336
+ "llm.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
337
+ "llm.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
338
+ "llm.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
339
+ "llm.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
340
+ "llm.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
341
+ "llm.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
342
+ "llm.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
343
+ "llm.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
344
+ "llm.model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
345
+ "llm.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
346
+ "llm.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
347
+ "llm.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
348
+ "llm.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
349
+ "llm.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
350
+ "llm.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
351
+ "llm.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
352
+ "llm.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
353
+ "llm.model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
354
+ "llm.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
355
+ "llm.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
356
+ "llm.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
357
+ "llm.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
358
+ "llm.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
359
+ "llm.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
360
+ "llm.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
361
+ "llm.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
362
+ "llm.model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
363
+ "llm.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
364
+ "llm.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
365
+ "llm.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
366
+ "llm.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
367
+ "llm.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
368
+ "llm.model.norm.weight": "model-00002-of-00002.safetensors",
369
+ "resampler.attn.in_proj_bias": "model-00002-of-00002.safetensors",
370
+ "resampler.attn.in_proj_weight": "model-00002-of-00002.safetensors",
371
+ "resampler.attn.out_proj.bias": "model-00002-of-00002.safetensors",
372
+ "resampler.attn.out_proj.weight": "model-00002-of-00002.safetensors",
373
+ "resampler.kv_proj.weight": "model-00002-of-00002.safetensors",
374
+ "resampler.ln_kv.bias": "model-00002-of-00002.safetensors",
375
+ "resampler.ln_kv.weight": "model-00002-of-00002.safetensors",
376
+ "resampler.ln_post.bias": "model-00002-of-00002.safetensors",
377
+ "resampler.ln_post.weight": "model-00002-of-00002.safetensors",
378
+ "resampler.ln_q.bias": "model-00002-of-00002.safetensors",
379
+ "resampler.ln_q.weight": "model-00002-of-00002.safetensors",
380
+ "resampler.pos_embed": "model-00002-of-00002.safetensors",
381
+ "resampler.proj": "model-00002-of-00002.safetensors",
382
+ "resampler.query": "model-00002-of-00002.safetensors",
383
+ "vpm.blocks.0.attn.proj.bias": "model-00002-of-00002.safetensors",
384
+ "vpm.blocks.0.attn.proj.weight": "model-00002-of-00002.safetensors",
385
+ "vpm.blocks.0.attn.qkv.bias": "model-00002-of-00002.safetensors",
386
+ "vpm.blocks.0.attn.qkv.weight": "model-00002-of-00002.safetensors",
387
+ "vpm.blocks.0.mlp.fc1.bias": "model-00002-of-00002.safetensors",
388
+ "vpm.blocks.0.mlp.fc1.weight": "model-00002-of-00002.safetensors",
389
+ "vpm.blocks.0.mlp.fc2.bias": "model-00002-of-00002.safetensors",
390
+ "vpm.blocks.0.mlp.fc2.weight": "model-00002-of-00002.safetensors",
391
+ "vpm.blocks.0.norm1.bias": "model-00002-of-00002.safetensors",
392
+ "vpm.blocks.0.norm1.weight": "model-00002-of-00002.safetensors",
393
+ "vpm.blocks.0.norm2.bias": "model-00002-of-00002.safetensors",
394
+ "vpm.blocks.0.norm2.weight": "model-00002-of-00002.safetensors",
395
+ "vpm.blocks.1.attn.proj.bias": "model-00002-of-00002.safetensors",
396
+ "vpm.blocks.1.attn.proj.weight": "model-00002-of-00002.safetensors",
397
+ "vpm.blocks.1.attn.qkv.bias": "model-00002-of-00002.safetensors",
398
+ "vpm.blocks.1.attn.qkv.weight": "model-00002-of-00002.safetensors",
399
+ "vpm.blocks.1.mlp.fc1.bias": "model-00002-of-00002.safetensors",
400
+ "vpm.blocks.1.mlp.fc1.weight": "model-00002-of-00002.safetensors",
401
+ "vpm.blocks.1.mlp.fc2.bias": "model-00002-of-00002.safetensors",
402
+ "vpm.blocks.1.mlp.fc2.weight": "model-00002-of-00002.safetensors",
403
+ "vpm.blocks.1.norm1.bias": "model-00002-of-00002.safetensors",
404
+ "vpm.blocks.1.norm1.weight": "model-00002-of-00002.safetensors",
405
+ "vpm.blocks.1.norm2.bias": "model-00002-of-00002.safetensors",
406
+ "vpm.blocks.1.norm2.weight": "model-00002-of-00002.safetensors",
407
+ "vpm.blocks.10.attn.proj.bias": "model-00002-of-00002.safetensors",
408
+ "vpm.blocks.10.attn.proj.weight": "model-00002-of-00002.safetensors",
409
+ "vpm.blocks.10.attn.qkv.bias": "model-00002-of-00002.safetensors",
410
+ "vpm.blocks.10.attn.qkv.weight": "model-00002-of-00002.safetensors",
411
+ "vpm.blocks.10.mlp.fc1.bias": "model-00002-of-00002.safetensors",
412
+ "vpm.blocks.10.mlp.fc1.weight": "model-00002-of-00002.safetensors",
413
+ "vpm.blocks.10.mlp.fc2.bias": "model-00002-of-00002.safetensors",
414
+ "vpm.blocks.10.mlp.fc2.weight": "model-00002-of-00002.safetensors",
415
+ "vpm.blocks.10.norm1.bias": "model-00002-of-00002.safetensors",
416
+ "vpm.blocks.10.norm1.weight": "model-00002-of-00002.safetensors",
417
+ "vpm.blocks.10.norm2.bias": "model-00002-of-00002.safetensors",
418
+ "vpm.blocks.10.norm2.weight": "model-00002-of-00002.safetensors",
419
+ "vpm.blocks.11.attn.proj.bias": "model-00002-of-00002.safetensors",
420
+ "vpm.blocks.11.attn.proj.weight": "model-00002-of-00002.safetensors",
421
+ "vpm.blocks.11.attn.qkv.bias": "model-00002-of-00002.safetensors",
422
+ "vpm.blocks.11.attn.qkv.weight": "model-00002-of-00002.safetensors",
423
+ "vpm.blocks.11.mlp.fc1.bias": "model-00002-of-00002.safetensors",
424
+ "vpm.blocks.11.mlp.fc1.weight": "model-00002-of-00002.safetensors",
425
+ "vpm.blocks.11.mlp.fc2.bias": "model-00002-of-00002.safetensors",
426
+ "vpm.blocks.11.mlp.fc2.weight": "model-00002-of-00002.safetensors",
427
+ "vpm.blocks.11.norm1.bias": "model-00002-of-00002.safetensors",
428
+ "vpm.blocks.11.norm1.weight": "model-00002-of-00002.safetensors",
429
+ "vpm.blocks.11.norm2.bias": "model-00002-of-00002.safetensors",
430
+ "vpm.blocks.11.norm2.weight": "model-00002-of-00002.safetensors",
431
+ "vpm.blocks.12.attn.proj.bias": "model-00002-of-00002.safetensors",
432
+ "vpm.blocks.12.attn.proj.weight": "model-00002-of-00002.safetensors",
433
+ "vpm.blocks.12.attn.qkv.bias": "model-00002-of-00002.safetensors",
434
+ "vpm.blocks.12.attn.qkv.weight": "model-00002-of-00002.safetensors",
435
+ "vpm.blocks.12.mlp.fc1.bias": "model-00002-of-00002.safetensors",
436
+ "vpm.blocks.12.mlp.fc1.weight": "model-00002-of-00002.safetensors",
437
+ "vpm.blocks.12.mlp.fc2.bias": "model-00002-of-00002.safetensors",
438
+ "vpm.blocks.12.mlp.fc2.weight": "model-00002-of-00002.safetensors",
439
+ "vpm.blocks.12.norm1.bias": "model-00002-of-00002.safetensors",
440
+ "vpm.blocks.12.norm1.weight": "model-00002-of-00002.safetensors",
441
+ "vpm.blocks.12.norm2.bias": "model-00002-of-00002.safetensors",
442
+ "vpm.blocks.12.norm2.weight": "model-00002-of-00002.safetensors",
443
+ "vpm.blocks.13.attn.proj.bias": "model-00002-of-00002.safetensors",
444
+ "vpm.blocks.13.attn.proj.weight": "model-00002-of-00002.safetensors",
445
+ "vpm.blocks.13.attn.qkv.bias": "model-00002-of-00002.safetensors",
446
+ "vpm.blocks.13.attn.qkv.weight": "model-00002-of-00002.safetensors",
447
+ "vpm.blocks.13.mlp.fc1.bias": "model-00002-of-00002.safetensors",
448
+ "vpm.blocks.13.mlp.fc1.weight": "model-00002-of-00002.safetensors",
449
+ "vpm.blocks.13.mlp.fc2.bias": "model-00002-of-00002.safetensors",
450
+ "vpm.blocks.13.mlp.fc2.weight": "model-00002-of-00002.safetensors",
451
+ "vpm.blocks.13.norm1.bias": "model-00002-of-00002.safetensors",
452
+ "vpm.blocks.13.norm1.weight": "model-00002-of-00002.safetensors",
453
+ "vpm.blocks.13.norm2.bias": "model-00002-of-00002.safetensors",
454
+ "vpm.blocks.13.norm2.weight": "model-00002-of-00002.safetensors",
455
+ "vpm.blocks.14.attn.proj.bias": "model-00002-of-00002.safetensors",
456
+ "vpm.blocks.14.attn.proj.weight": "model-00002-of-00002.safetensors",
457
+ "vpm.blocks.14.attn.qkv.bias": "model-00002-of-00002.safetensors",
458
+ "vpm.blocks.14.attn.qkv.weight": "model-00002-of-00002.safetensors",
459
+ "vpm.blocks.14.mlp.fc1.bias": "model-00002-of-00002.safetensors",
460
+ "vpm.blocks.14.mlp.fc1.weight": "model-00002-of-00002.safetensors",
461
+ "vpm.blocks.14.mlp.fc2.bias": "model-00002-of-00002.safetensors",
462
+ "vpm.blocks.14.mlp.fc2.weight": "model-00002-of-00002.safetensors",
463
+ "vpm.blocks.14.norm1.bias": "model-00002-of-00002.safetensors",
464
+ "vpm.blocks.14.norm1.weight": "model-00002-of-00002.safetensors",
465
+ "vpm.blocks.14.norm2.bias": "model-00002-of-00002.safetensors",
466
+ "vpm.blocks.14.norm2.weight": "model-00002-of-00002.safetensors",
467
+ "vpm.blocks.15.attn.proj.bias": "model-00002-of-00002.safetensors",
468
+ "vpm.blocks.15.attn.proj.weight": "model-00002-of-00002.safetensors",
469
+ "vpm.blocks.15.attn.qkv.bias": "model-00002-of-00002.safetensors",
470
+ "vpm.blocks.15.attn.qkv.weight": "model-00002-of-00002.safetensors",
471
+ "vpm.blocks.15.mlp.fc1.bias": "model-00002-of-00002.safetensors",
472
+ "vpm.blocks.15.mlp.fc1.weight": "model-00002-of-00002.safetensors",
473
+ "vpm.blocks.15.mlp.fc2.bias": "model-00002-of-00002.safetensors",
474
+ "vpm.blocks.15.mlp.fc2.weight": "model-00002-of-00002.safetensors",
475
+ "vpm.blocks.15.norm1.bias": "model-00002-of-00002.safetensors",
476
+ "vpm.blocks.15.norm1.weight": "model-00002-of-00002.safetensors",
477
+ "vpm.blocks.15.norm2.bias": "model-00002-of-00002.safetensors",
478
+ "vpm.blocks.15.norm2.weight": "model-00002-of-00002.safetensors",
479
+ "vpm.blocks.16.attn.proj.bias": "model-00002-of-00002.safetensors",
480
+ "vpm.blocks.16.attn.proj.weight": "model-00002-of-00002.safetensors",
481
+ "vpm.blocks.16.attn.qkv.bias": "model-00002-of-00002.safetensors",
482
+ "vpm.blocks.16.attn.qkv.weight": "model-00002-of-00002.safetensors",
483
+ "vpm.blocks.16.mlp.fc1.bias": "model-00002-of-00002.safetensors",
484
+ "vpm.blocks.16.mlp.fc1.weight": "model-00002-of-00002.safetensors",
485
+ "vpm.blocks.16.mlp.fc2.bias": "model-00002-of-00002.safetensors",
486
+ "vpm.blocks.16.mlp.fc2.weight": "model-00002-of-00002.safetensors",
487
+ "vpm.blocks.16.norm1.bias": "model-00002-of-00002.safetensors",
488
+ "vpm.blocks.16.norm1.weight": "model-00002-of-00002.safetensors",
489
+ "vpm.blocks.16.norm2.bias": "model-00002-of-00002.safetensors",
490
+ "vpm.blocks.16.norm2.weight": "model-00002-of-00002.safetensors",
491
+ "vpm.blocks.17.attn.proj.bias": "model-00002-of-00002.safetensors",
492
+ "vpm.blocks.17.attn.proj.weight": "model-00002-of-00002.safetensors",
493
+ "vpm.blocks.17.attn.qkv.bias": "model-00002-of-00002.safetensors",
494
+ "vpm.blocks.17.attn.qkv.weight": "model-00002-of-00002.safetensors",
495
+ "vpm.blocks.17.mlp.fc1.bias": "model-00002-of-00002.safetensors",
496
+ "vpm.blocks.17.mlp.fc1.weight": "model-00002-of-00002.safetensors",
497
+ "vpm.blocks.17.mlp.fc2.bias": "model-00002-of-00002.safetensors",
498
+ "vpm.blocks.17.mlp.fc2.weight": "model-00002-of-00002.safetensors",
499
+ "vpm.blocks.17.norm1.bias": "model-00002-of-00002.safetensors",
500
+ "vpm.blocks.17.norm1.weight": "model-00002-of-00002.safetensors",
501
+ "vpm.blocks.17.norm2.bias": "model-00002-of-00002.safetensors",
502
+ "vpm.blocks.17.norm2.weight": "model-00002-of-00002.safetensors",
503
+ "vpm.blocks.18.attn.proj.bias": "model-00002-of-00002.safetensors",
504
+ "vpm.blocks.18.attn.proj.weight": "model-00002-of-00002.safetensors",
505
+ "vpm.blocks.18.attn.qkv.bias": "model-00002-of-00002.safetensors",
506
+ "vpm.blocks.18.attn.qkv.weight": "model-00002-of-00002.safetensors",
507
+ "vpm.blocks.18.mlp.fc1.bias": "model-00002-of-00002.safetensors",
508
+ "vpm.blocks.18.mlp.fc1.weight": "model-00002-of-00002.safetensors",
509
+ "vpm.blocks.18.mlp.fc2.bias": "model-00002-of-00002.safetensors",
510
+ "vpm.blocks.18.mlp.fc2.weight": "model-00002-of-00002.safetensors",
511
+ "vpm.blocks.18.norm1.bias": "model-00002-of-00002.safetensors",
512
+ "vpm.blocks.18.norm1.weight": "model-00002-of-00002.safetensors",
513
+ "vpm.blocks.18.norm2.bias": "model-00002-of-00002.safetensors",
514
+ "vpm.blocks.18.norm2.weight": "model-00002-of-00002.safetensors",
515
+ "vpm.blocks.19.attn.proj.bias": "model-00002-of-00002.safetensors",
516
+ "vpm.blocks.19.attn.proj.weight": "model-00002-of-00002.safetensors",
517
+ "vpm.blocks.19.attn.qkv.bias": "model-00002-of-00002.safetensors",
518
+ "vpm.blocks.19.attn.qkv.weight": "model-00002-of-00002.safetensors",
519
+ "vpm.blocks.19.mlp.fc1.bias": "model-00002-of-00002.safetensors",
520
+ "vpm.blocks.19.mlp.fc1.weight": "model-00002-of-00002.safetensors",
521
+ "vpm.blocks.19.mlp.fc2.bias": "model-00002-of-00002.safetensors",
522
+ "vpm.blocks.19.mlp.fc2.weight": "model-00002-of-00002.safetensors",
523
+ "vpm.blocks.19.norm1.bias": "model-00002-of-00002.safetensors",
524
+ "vpm.blocks.19.norm1.weight": "model-00002-of-00002.safetensors",
525
+ "vpm.blocks.19.norm2.bias": "model-00002-of-00002.safetensors",
526
+ "vpm.blocks.19.norm2.weight": "model-00002-of-00002.safetensors",
527
+ "vpm.blocks.2.attn.proj.bias": "model-00002-of-00002.safetensors",
528
+ "vpm.blocks.2.attn.proj.weight": "model-00002-of-00002.safetensors",
529
+ "vpm.blocks.2.attn.qkv.bias": "model-00002-of-00002.safetensors",
530
+ "vpm.blocks.2.attn.qkv.weight": "model-00002-of-00002.safetensors",
531
+ "vpm.blocks.2.mlp.fc1.bias": "model-00002-of-00002.safetensors",
532
+ "vpm.blocks.2.mlp.fc1.weight": "model-00002-of-00002.safetensors",
533
+ "vpm.blocks.2.mlp.fc2.bias": "model-00002-of-00002.safetensors",
534
+ "vpm.blocks.2.mlp.fc2.weight": "model-00002-of-00002.safetensors",
535
+ "vpm.blocks.2.norm1.bias": "model-00002-of-00002.safetensors",
536
+ "vpm.blocks.2.norm1.weight": "model-00002-of-00002.safetensors",
537
+ "vpm.blocks.2.norm2.bias": "model-00002-of-00002.safetensors",
538
+ "vpm.blocks.2.norm2.weight": "model-00002-of-00002.safetensors",
539
+ "vpm.blocks.20.attn.proj.bias": "model-00002-of-00002.safetensors",
540
+ "vpm.blocks.20.attn.proj.weight": "model-00002-of-00002.safetensors",
541
+ "vpm.blocks.20.attn.qkv.bias": "model-00002-of-00002.safetensors",
542
+ "vpm.blocks.20.attn.qkv.weight": "model-00002-of-00002.safetensors",
543
+ "vpm.blocks.20.mlp.fc1.bias": "model-00002-of-00002.safetensors",
544
+ "vpm.blocks.20.mlp.fc1.weight": "model-00002-of-00002.safetensors",
545
+ "vpm.blocks.20.mlp.fc2.bias": "model-00002-of-00002.safetensors",
546
+ "vpm.blocks.20.mlp.fc2.weight": "model-00002-of-00002.safetensors",
547
+ "vpm.blocks.20.norm1.bias": "model-00002-of-00002.safetensors",
548
+ "vpm.blocks.20.norm1.weight": "model-00002-of-00002.safetensors",
549
+ "vpm.blocks.20.norm2.bias": "model-00002-of-00002.safetensors",
550
+ "vpm.blocks.20.norm2.weight": "model-00002-of-00002.safetensors",
551
+ "vpm.blocks.21.attn.proj.bias": "model-00002-of-00002.safetensors",
552
+ "vpm.blocks.21.attn.proj.weight": "model-00002-of-00002.safetensors",
553
+ "vpm.blocks.21.attn.qkv.bias": "model-00002-of-00002.safetensors",
554
+ "vpm.blocks.21.attn.qkv.weight": "model-00002-of-00002.safetensors",
555
+ "vpm.blocks.21.mlp.fc1.bias": "model-00002-of-00002.safetensors",
556
+ "vpm.blocks.21.mlp.fc1.weight": "model-00002-of-00002.safetensors",
557
+ "vpm.blocks.21.mlp.fc2.bias": "model-00002-of-00002.safetensors",
558
+ "vpm.blocks.21.mlp.fc2.weight": "model-00002-of-00002.safetensors",
559
+ "vpm.blocks.21.norm1.bias": "model-00002-of-00002.safetensors",
560
+ "vpm.blocks.21.norm1.weight": "model-00002-of-00002.safetensors",
561
+ "vpm.blocks.21.norm2.bias": "model-00002-of-00002.safetensors",
562
+ "vpm.blocks.21.norm2.weight": "model-00002-of-00002.safetensors",
563
+ "vpm.blocks.22.attn.proj.bias": "model-00002-of-00002.safetensors",
564
+ "vpm.blocks.22.attn.proj.weight": "model-00002-of-00002.safetensors",
565
+ "vpm.blocks.22.attn.qkv.bias": "model-00002-of-00002.safetensors",
566
+ "vpm.blocks.22.attn.qkv.weight": "model-00002-of-00002.safetensors",
567
+ "vpm.blocks.22.mlp.fc1.bias": "model-00002-of-00002.safetensors",
568
+ "vpm.blocks.22.mlp.fc1.weight": "model-00002-of-00002.safetensors",
569
+ "vpm.blocks.22.mlp.fc2.bias": "model-00002-of-00002.safetensors",
570
+ "vpm.blocks.22.mlp.fc2.weight": "model-00002-of-00002.safetensors",
571
+ "vpm.blocks.22.norm1.bias": "model-00002-of-00002.safetensors",
572
+ "vpm.blocks.22.norm1.weight": "model-00002-of-00002.safetensors",
573
+ "vpm.blocks.22.norm2.bias": "model-00002-of-00002.safetensors",
574
+ "vpm.blocks.22.norm2.weight": "model-00002-of-00002.safetensors",
575
+ "vpm.blocks.23.attn.proj.bias": "model-00002-of-00002.safetensors",
576
+ "vpm.blocks.23.attn.proj.weight": "model-00002-of-00002.safetensors",
577
+ "vpm.blocks.23.attn.qkv.bias": "model-00002-of-00002.safetensors",
578
+ "vpm.blocks.23.attn.qkv.weight": "model-00002-of-00002.safetensors",
579
+ "vpm.blocks.23.mlp.fc1.bias": "model-00002-of-00002.safetensors",
580
+ "vpm.blocks.23.mlp.fc1.weight": "model-00002-of-00002.safetensors",
581
+ "vpm.blocks.23.mlp.fc2.bias": "model-00002-of-00002.safetensors",
582
+ "vpm.blocks.23.mlp.fc2.weight": "model-00002-of-00002.safetensors",
583
+ "vpm.blocks.23.norm1.bias": "model-00002-of-00002.safetensors",
584
+ "vpm.blocks.23.norm1.weight": "model-00002-of-00002.safetensors",
585
+ "vpm.blocks.23.norm2.bias": "model-00002-of-00002.safetensors",
586
+ "vpm.blocks.23.norm2.weight": "model-00002-of-00002.safetensors",
587
+ "vpm.blocks.24.attn.proj.bias": "model-00002-of-00002.safetensors",
588
+ "vpm.blocks.24.attn.proj.weight": "model-00002-of-00002.safetensors",
589
+ "vpm.blocks.24.attn.qkv.bias": "model-00002-of-00002.safetensors",
590
+ "vpm.blocks.24.attn.qkv.weight": "model-00002-of-00002.safetensors",
591
+ "vpm.blocks.24.mlp.fc1.bias": "model-00002-of-00002.safetensors",
592
+ "vpm.blocks.24.mlp.fc1.weight": "model-00002-of-00002.safetensors",
593
+ "vpm.blocks.24.mlp.fc2.bias": "model-00002-of-00002.safetensors",
594
+ "vpm.blocks.24.mlp.fc2.weight": "model-00002-of-00002.safetensors",
595
+ "vpm.blocks.24.norm1.bias": "model-00002-of-00002.safetensors",
596
+ "vpm.blocks.24.norm1.weight": "model-00002-of-00002.safetensors",
597
+ "vpm.blocks.24.norm2.bias": "model-00002-of-00002.safetensors",
598
+ "vpm.blocks.24.norm2.weight": "model-00002-of-00002.safetensors",
599
+ "vpm.blocks.25.attn.proj.bias": "model-00002-of-00002.safetensors",
600
+ "vpm.blocks.25.attn.proj.weight": "model-00002-of-00002.safetensors",
601
+ "vpm.blocks.25.attn.qkv.bias": "model-00002-of-00002.safetensors",
602
+ "vpm.blocks.25.attn.qkv.weight": "model-00002-of-00002.safetensors",
603
+ "vpm.blocks.25.mlp.fc1.bias": "model-00002-of-00002.safetensors",
604
+ "vpm.blocks.25.mlp.fc1.weight": "model-00002-of-00002.safetensors",
605
+ "vpm.blocks.25.mlp.fc2.bias": "model-00002-of-00002.safetensors",
606
+ "vpm.blocks.25.mlp.fc2.weight": "model-00002-of-00002.safetensors",
607
+ "vpm.blocks.25.norm1.bias": "model-00002-of-00002.safetensors",
608
+ "vpm.blocks.25.norm1.weight": "model-00002-of-00002.safetensors",
609
+ "vpm.blocks.25.norm2.bias": "model-00002-of-00002.safetensors",
610
+ "vpm.blocks.25.norm2.weight": "model-00002-of-00002.safetensors",
611
+ "vpm.blocks.3.attn.proj.bias": "model-00002-of-00002.safetensors",
612
+ "vpm.blocks.3.attn.proj.weight": "model-00002-of-00002.safetensors",
613
+ "vpm.blocks.3.attn.qkv.bias": "model-00002-of-00002.safetensors",
614
+ "vpm.blocks.3.attn.qkv.weight": "model-00002-of-00002.safetensors",
615
+ "vpm.blocks.3.mlp.fc1.bias": "model-00002-of-00002.safetensors",
616
+ "vpm.blocks.3.mlp.fc1.weight": "model-00002-of-00002.safetensors",
617
+ "vpm.blocks.3.mlp.fc2.bias": "model-00002-of-00002.safetensors",
618
+ "vpm.blocks.3.mlp.fc2.weight": "model-00002-of-00002.safetensors",
619
+ "vpm.blocks.3.norm1.bias": "model-00002-of-00002.safetensors",
620
+ "vpm.blocks.3.norm1.weight": "model-00002-of-00002.safetensors",
621
+ "vpm.blocks.3.norm2.bias": "model-00002-of-00002.safetensors",
622
+ "vpm.blocks.3.norm2.weight": "model-00002-of-00002.safetensors",
623
+ "vpm.blocks.4.attn.proj.bias": "model-00002-of-00002.safetensors",
624
+ "vpm.blocks.4.attn.proj.weight": "model-00002-of-00002.safetensors",
625
+ "vpm.blocks.4.attn.qkv.bias": "model-00002-of-00002.safetensors",
626
+ "vpm.blocks.4.attn.qkv.weight": "model-00002-of-00002.safetensors",
627
+ "vpm.blocks.4.mlp.fc1.bias": "model-00002-of-00002.safetensors",
628
+ "vpm.blocks.4.mlp.fc1.weight": "model-00002-of-00002.safetensors",
629
+ "vpm.blocks.4.mlp.fc2.bias": "model-00002-of-00002.safetensors",
630
+ "vpm.blocks.4.mlp.fc2.weight": "model-00002-of-00002.safetensors",
631
+ "vpm.blocks.4.norm1.bias": "model-00002-of-00002.safetensors",
632
+ "vpm.blocks.4.norm1.weight": "model-00002-of-00002.safetensors",
633
+ "vpm.blocks.4.norm2.bias": "model-00002-of-00002.safetensors",
634
+ "vpm.blocks.4.norm2.weight": "model-00002-of-00002.safetensors",
635
+ "vpm.blocks.5.attn.proj.bias": "model-00002-of-00002.safetensors",
636
+ "vpm.blocks.5.attn.proj.weight": "model-00002-of-00002.safetensors",
637
+ "vpm.blocks.5.attn.qkv.bias": "model-00002-of-00002.safetensors",
638
+ "vpm.blocks.5.attn.qkv.weight": "model-00002-of-00002.safetensors",
639
+ "vpm.blocks.5.mlp.fc1.bias": "model-00002-of-00002.safetensors",
640
+ "vpm.blocks.5.mlp.fc1.weight": "model-00002-of-00002.safetensors",
641
+ "vpm.blocks.5.mlp.fc2.bias": "model-00002-of-00002.safetensors",
642
+ "vpm.blocks.5.mlp.fc2.weight": "model-00002-of-00002.safetensors",
643
+ "vpm.blocks.5.norm1.bias": "model-00002-of-00002.safetensors",
644
+ "vpm.blocks.5.norm1.weight": "model-00002-of-00002.safetensors",
645
+ "vpm.blocks.5.norm2.bias": "model-00002-of-00002.safetensors",
646
+ "vpm.blocks.5.norm2.weight": "model-00002-of-00002.safetensors",
647
+ "vpm.blocks.6.attn.proj.bias": "model-00002-of-00002.safetensors",
648
+ "vpm.blocks.6.attn.proj.weight": "model-00002-of-00002.safetensors",
649
+ "vpm.blocks.6.attn.qkv.bias": "model-00002-of-00002.safetensors",
650
+ "vpm.blocks.6.attn.qkv.weight": "model-00002-of-00002.safetensors",
651
+ "vpm.blocks.6.mlp.fc1.bias": "model-00002-of-00002.safetensors",
652
+ "vpm.blocks.6.mlp.fc1.weight": "model-00002-of-00002.safetensors",
653
+ "vpm.blocks.6.mlp.fc2.bias": "model-00002-of-00002.safetensors",
654
+ "vpm.blocks.6.mlp.fc2.weight": "model-00002-of-00002.safetensors",
655
+ "vpm.blocks.6.norm1.bias": "model-00002-of-00002.safetensors",
656
+ "vpm.blocks.6.norm1.weight": "model-00002-of-00002.safetensors",
657
+ "vpm.blocks.6.norm2.bias": "model-00002-of-00002.safetensors",
658
+ "vpm.blocks.6.norm2.weight": "model-00002-of-00002.safetensors",
659
+ "vpm.blocks.7.attn.proj.bias": "model-00002-of-00002.safetensors",
660
+ "vpm.blocks.7.attn.proj.weight": "model-00002-of-00002.safetensors",
661
+ "vpm.blocks.7.attn.qkv.bias": "model-00002-of-00002.safetensors",
662
+ "vpm.blocks.7.attn.qkv.weight": "model-00002-of-00002.safetensors",
663
+ "vpm.blocks.7.mlp.fc1.bias": "model-00002-of-00002.safetensors",
664
+ "vpm.blocks.7.mlp.fc1.weight": "model-00002-of-00002.safetensors",
665
+ "vpm.blocks.7.mlp.fc2.bias": "model-00002-of-00002.safetensors",
666
+ "vpm.blocks.7.mlp.fc2.weight": "model-00002-of-00002.safetensors",
667
+ "vpm.blocks.7.norm1.bias": "model-00002-of-00002.safetensors",
668
+ "vpm.blocks.7.norm1.weight": "model-00002-of-00002.safetensors",
669
+ "vpm.blocks.7.norm2.bias": "model-00002-of-00002.safetensors",
670
+ "vpm.blocks.7.norm2.weight": "model-00002-of-00002.safetensors",
671
+ "vpm.blocks.8.attn.proj.bias": "model-00002-of-00002.safetensors",
672
+ "vpm.blocks.8.attn.proj.weight": "model-00002-of-00002.safetensors",
673
+ "vpm.blocks.8.attn.qkv.bias": "model-00002-of-00002.safetensors",
674
+ "vpm.blocks.8.attn.qkv.weight": "model-00002-of-00002.safetensors",
675
+ "vpm.blocks.8.mlp.fc1.bias": "model-00002-of-00002.safetensors",
676
+ "vpm.blocks.8.mlp.fc1.weight": "model-00002-of-00002.safetensors",
677
+ "vpm.blocks.8.mlp.fc2.bias": "model-00002-of-00002.safetensors",
678
+ "vpm.blocks.8.mlp.fc2.weight": "model-00002-of-00002.safetensors",
679
+ "vpm.blocks.8.norm1.bias": "model-00002-of-00002.safetensors",
680
+ "vpm.blocks.8.norm1.weight": "model-00002-of-00002.safetensors",
681
+ "vpm.blocks.8.norm2.bias": "model-00002-of-00002.safetensors",
682
+ "vpm.blocks.8.norm2.weight": "model-00002-of-00002.safetensors",
683
+ "vpm.blocks.9.attn.proj.bias": "model-00002-of-00002.safetensors",
684
+ "vpm.blocks.9.attn.proj.weight": "model-00002-of-00002.safetensors",
685
+ "vpm.blocks.9.attn.qkv.bias": "model-00002-of-00002.safetensors",
686
+ "vpm.blocks.9.attn.qkv.weight": "model-00002-of-00002.safetensors",
687
+ "vpm.blocks.9.mlp.fc1.bias": "model-00002-of-00002.safetensors",
688
+ "vpm.blocks.9.mlp.fc1.weight": "model-00002-of-00002.safetensors",
689
+ "vpm.blocks.9.mlp.fc2.bias": "model-00002-of-00002.safetensors",
690
+ "vpm.blocks.9.mlp.fc2.weight": "model-00002-of-00002.safetensors",
691
+ "vpm.blocks.9.norm1.bias": "model-00002-of-00002.safetensors",
692
+ "vpm.blocks.9.norm1.weight": "model-00002-of-00002.safetensors",
693
+ "vpm.blocks.9.norm2.bias": "model-00002-of-00002.safetensors",
694
+ "vpm.blocks.9.norm2.weight": "model-00002-of-00002.safetensors",
695
+ "vpm.norm.bias": "model-00002-of-00002.safetensors",
696
+ "vpm.norm.weight": "model-00002-of-00002.safetensors",
697
+ "vpm.patch_embed.proj.bias": "model-00002-of-00002.safetensors",
698
+ "vpm.patch_embed.proj.weight": "model-00002-of-00002.safetensors",
699
+ "vpm.pos_embed": "model-00002-of-00002.safetensors"
700
+ }
701
+ }
modeling_minicpm.py ADDED
@@ -0,0 +1,1697 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+ # and OPT implementations in this library. It has been modified from its
6
+ # original forms to accommodate minor architectural differences compared
7
+ # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+ """ PyTorch MiniCPM model."""
21
+ import math
22
+ import re
23
+ import warnings
24
+ from typing import Dict, List, Optional, Tuple, Union
25
+
26
+ import torch
27
+ import torch.nn.functional as F
28
+ import torch.utils.checkpoint
29
+ from torch import nn
30
+ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
31
+ from transformers.activations import ACT2FN
32
+ from transformers.cache_utils import Cache, DynamicCache
33
+ from transformers.modeling_attn_mask_utils import (
34
+ AttentionMaskConverter,
35
+ _prepare_4d_attention_mask,
36
+ _prepare_4d_causal_attention_mask,
37
+ _prepare_4d_causal_attention_mask_for_sdpa,
38
+ )
39
+ from transformers.modeling_outputs import (
40
+ BaseModelOutputWithPast,
41
+ CausalLMOutputWithPast,
42
+ SequenceClassifierOutputWithPast,
43
+ )
44
+ from transformers.modeling_utils import PreTrainedModel
45
+ from transformers.pytorch_utils import (
46
+ ALL_LAYERNORM_LAYERS,
47
+ is_torch_greater_or_equal_than_1_13,
48
+ )
49
+ from transformers.utils import (
50
+ add_start_docstrings,
51
+ add_start_docstrings_to_model_forward,
52
+ is_flash_attn_2_available,
53
+ is_flash_attn_greater_or_equal_2_10,
54
+ logging,
55
+ replace_return_docstrings,
56
+ )
57
+ from transformers.utils.import_utils import is_torch_fx_available
58
+
59
+ from .configuration_minicpm import MiniCPMConfig
60
+
61
+ try:
62
+ from flash_attn import flash_attn_func, flash_attn_varlen_func
63
+ from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa
64
+ except:
65
+ pass
66
+
67
+ # This makes `_prepare_4d_causal_attention_mask` a leaf function in the FX graph.
68
+ # It means that the function will not be traced through and simply appear as a node in the graph.
69
+ if is_torch_fx_available():
70
+ if not is_torch_greater_or_equal_than_1_13:
71
+ import torch.fx
72
+
73
+ _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask)
74
+
75
+ logger = logging.get_logger(__name__)
76
+
77
+ _CONFIG_FOR_DOC = "MiniCPMConfig"
78
+
79
+
80
+ def _get_unpad_data(attention_mask):
81
+ seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
82
+ indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
83
+ max_seqlen_in_batch = seqlens_in_batch.max().item()
84
+ cu_seqlens = F.pad(
85
+ torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0)
86
+ )
87
+ return (
88
+ indices,
89
+ cu_seqlens,
90
+ max_seqlen_in_batch,
91
+ )
92
+
93
+
94
+ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
95
+ warnings.warn(
96
+ "Calling `transformers.models.minicpm.modeling_minicpm._prepare_4d_attention_mask` is deprecated and will be removed in v4.37. Use `transformers.modeling_attn_mask_utils._prepare_4d_attention_mask"
97
+ )
98
+ return _prepare_4d_attention_mask(mask=mask, dtype=dtype, tgt_len=tgt_len)
99
+
100
+
101
+ def _make_causal_mask(
102
+ input_ids_shape: torch.Size,
103
+ dtype: torch.dtype,
104
+ device: torch.device,
105
+ past_key_values_length: int = 0,
106
+ ):
107
+ warnings.warn(
108
+ "Calling `transformers.models.minicpm.modeling_minicpm._make_causal_mask` is deprecated and will be removed in v4.37. Use `transformers.models.minicpm.modeling_minicpm.AttentionMaskConverter._make_causal_mask"
109
+ )
110
+ return AttentionMaskConverter._make_causal_mask(
111
+ input_ids_shape=input_ids_shape,
112
+ dtype=dtype,
113
+ device=device,
114
+ past_key_values_length=past_key_values_length,
115
+ )
116
+
117
+
118
+ # @torch.jit.script # type: ignore
119
+ def rms_layernorm(hidden: torch.Tensor, weight: torch.Tensor, eps: float):
120
+ old_dtype = hidden.dtype
121
+ variance = hidden.to(torch.float32).pow(2).mean(dim=-1, keepdim=True)
122
+ hidden = (hidden * torch.rsqrt(variance + eps)).to(old_dtype)
123
+ return hidden * weight
124
+
125
+
126
+ class MiniCPMRMSNorm(nn.Module):
127
+ def __init__(self, hidden_size, eps=1e-6):
128
+ """
129
+ MiniCPMRMSNorm is equivalent to T5LayerNorm
130
+ """
131
+ super().__init__()
132
+ self.weight = nn.Parameter(torch.ones(hidden_size))
133
+ self.variance_epsilon = eps
134
+
135
+ def forward(self, hidden_states):
136
+ return rms_layernorm(hidden_states, self.weight, self.variance_epsilon)
137
+
138
+
139
+ ALL_LAYERNORM_LAYERS.append(MiniCPMRMSNorm)
140
+
141
+
142
+ class MiniCPMRotaryEmbedding(nn.Module):
143
+ def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
144
+ super().__init__()
145
+
146
+ self.dim = dim
147
+ self.max_position_embeddings = max_position_embeddings
148
+ self.base = base
149
+ inv_freq = 1.0 / (
150
+ self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
151
+ )
152
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
153
+
154
+ # Build here to make `torch.jit.trace` work.
155
+ self._set_cos_sin_cache(
156
+ # seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
157
+ seq_len=max_position_embeddings,
158
+ device=self.inv_freq.device,
159
+ dtype=torch.float32,
160
+ )
161
+
162
+ def _set_cos_sin_cache(self, seq_len, device, dtype):
163
+ self.max_seq_len_cached = seq_len
164
+ t = torch.arange(
165
+ self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype
166
+ )
167
+ freqs = torch.outer(t, self.inv_freq)
168
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
169
+ emb = torch.cat((freqs, freqs), dim=-1)
170
+
171
+ self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
172
+ self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
173
+
174
+ def forward(self, x, seq_len=None):
175
+ # x: [bs, num_attention_heads, seq_len, head_size]
176
+ if seq_len > self.max_seq_len_cached:
177
+ self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
178
+
179
+ return (
180
+ self.cos_cached[:seq_len].to(dtype=x.dtype),
181
+ self.sin_cached[:seq_len].to(dtype=x.dtype),
182
+ )
183
+
184
+
185
+ class MiniCPMLinearScalingRotaryEmbedding(MiniCPMRotaryEmbedding):
186
+ """MiniCPMRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
187
+
188
+ def __init__(
189
+ self,
190
+ dim,
191
+ max_position_embeddings=2048,
192
+ base=10000,
193
+ device=None,
194
+ scaling_factor=1.0,
195
+ ):
196
+ self.scaling_factor = scaling_factor
197
+ super().__init__(dim, max_position_embeddings, base, device)
198
+
199
+ def _set_cos_sin_cache(self, seq_len, device, dtype):
200
+ self.max_seq_len_cached = seq_len
201
+ t = torch.arange(
202
+ self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype
203
+ )
204
+ t = t / self.scaling_factor
205
+
206
+ freqs = torch.outer(t, self.inv_freq)
207
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
208
+ emb = torch.cat((freqs, freqs), dim=-1)
209
+ self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
210
+ self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
211
+
212
+
213
+ class MiniCPMDynamicNTKScalingRotaryEmbedding(MiniCPMRotaryEmbedding):
214
+ """MiniCPMRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
215
+
216
+ def __init__(
217
+ self,
218
+ dim,
219
+ max_position_embeddings=2048,
220
+ base=10000,
221
+ device=None,
222
+ scaling_factor=1.0,
223
+ ):
224
+ self.scaling_factor = scaling_factor
225
+ super().__init__(dim, max_position_embeddings, base, device)
226
+
227
+ def _set_cos_sin_cache(self, seq_len, device, dtype):
228
+ self.max_seq_len_cached = seq_len
229
+
230
+ if seq_len > self.max_position_embeddings:
231
+ base = self.base * (
232
+ (self.scaling_factor * seq_len / self.max_position_embeddings)
233
+ - (self.scaling_factor - 1)
234
+ ) ** (self.dim / (self.dim - 2))
235
+ inv_freq = 1.0 / (
236
+ base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
237
+ )
238
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
239
+
240
+ t = torch.arange(
241
+ self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype
242
+ )
243
+
244
+ freqs = torch.outer(t, self.inv_freq)
245
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
246
+ emb = torch.cat((freqs, freqs), dim=-1)
247
+
248
+ self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
249
+ self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
250
+
251
+
252
+ def rotate_half(x):
253
+ """Rotates half the hidden dims of the input."""
254
+ x1 = x[..., : x.shape[-1] // 2]
255
+ x2 = x[..., x.shape[-1] // 2 :]
256
+ return torch.cat((-x2, x1), dim=-1)
257
+
258
+
259
+ def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
260
+ """Applies Rotary Position Embedding to the query and key tensors.
261
+ Args:
262
+ q (`torch.Tensor`): The query tensor.
263
+ k (`torch.Tensor`): The key tensor.
264
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
265
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
266
+ position_ids (`torch.Tensor`):
267
+ The position indices of the tokens corresponding to the query and key tensors. For example, this can be
268
+ used to pass offsetted position ids when working with a KV-cache.
269
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
270
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
271
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
272
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
273
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
274
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
275
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
276
+ Returns:
277
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
278
+ """
279
+ # cos = cos[position_ids].unsqueeze(unsqueeze_dim)
280
+ # sin = sin[position_ids].unsqueeze(unsqueeze_dim)
281
+ # q_embed = (q * cos) + (rotate_half(q) * sin)
282
+ # k_embed = (k * cos) + (rotate_half(k) * sin)
283
+ orig_dtype = k.dtype
284
+ cos = cos[position_ids].unsqueeze(unsqueeze_dim) # [bs, 1, seq_len, dim]
285
+ sin = sin[position_ids].unsqueeze(unsqueeze_dim) # [bs, 1, seq_len, dim]
286
+ q_fp32 = q.to(dtype=torch.float32, device=q.device)
287
+ k_fp32 = k.to(dtype=torch.float32, device=k.device)
288
+ q_embed = (q_fp32 * cos) + (rotate_half(q_fp32) * sin)
289
+ k_embed = (k_fp32 * cos) + (rotate_half(k_fp32) * sin)
290
+ return q_embed.to(dtype=orig_dtype), k_embed.to(dtype=orig_dtype)
291
+
292
+
293
+ class MiniCPMMLP(nn.Module):
294
+ def __init__(self, config):
295
+ super().__init__()
296
+ self.config = config
297
+ self.hidden_size = config.hidden_size
298
+ self.intermediate_size = config.intermediate_size
299
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
300
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
301
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
302
+ self.act_fn = ACT2FN[config.hidden_act]
303
+
304
+ def forward(self, x):
305
+ if self.config.pretraining_tp > 1:
306
+ slice = self.intermediate_size // self.config.pretraining_tp
307
+ gate_proj_slices = self.gate_proj.weight.split(slice, dim=0)
308
+ up_proj_slices = self.up_proj.weight.split(slice, dim=0)
309
+ down_proj_slices = self.down_proj.weight.split(slice, dim=1)
310
+
311
+ gate_proj = torch.cat(
312
+ [
313
+ F.linear(x, gate_proj_slices[i])
314
+ for i in range(self.config.pretraining_tp)
315
+ ],
316
+ dim=-1,
317
+ )
318
+ up_proj = torch.cat(
319
+ [
320
+ F.linear(x, up_proj_slices[i])
321
+ for i in range(self.config.pretraining_tp)
322
+ ],
323
+ dim=-1,
324
+ )
325
+
326
+ intermediate_states = (self.act_fn(gate_proj) * up_proj).split(slice, dim=2)
327
+ down_proj = [
328
+ F.linear(intermediate_states[i], down_proj_slices[i])
329
+ for i in range(self.config.pretraining_tp)
330
+ ]
331
+ down_proj = sum(down_proj)
332
+ else:
333
+ down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
334
+
335
+ return down_proj
336
+
337
+
338
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
339
+ """
340
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
341
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
342
+ """
343
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
344
+ if n_rep == 1:
345
+ return hidden_states
346
+ hidden_states = hidden_states[:, :, None, :, :].expand(
347
+ batch, num_key_value_heads, n_rep, slen, head_dim
348
+ )
349
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
350
+
351
+
352
+ class MiniCPMAttention(nn.Module):
353
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
354
+
355
+ def __init__(self, config: MiniCPMConfig, layer_idx: Optional[int] = None):
356
+ super().__init__()
357
+ self.config = config
358
+ self.layer_idx = layer_idx
359
+ if layer_idx is None:
360
+ logger.warning_once(
361
+ f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
362
+ "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
363
+ "when creating this class."
364
+ )
365
+
366
+ self.attention_dropout = config.attention_dropout
367
+ self.hidden_size = config.hidden_size
368
+ self.num_heads = config.num_attention_heads
369
+ self.head_dim = self.hidden_size // self.num_heads
370
+ self.num_key_value_heads = config.num_key_value_heads
371
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
372
+ self.max_position_embeddings = config.max_position_embeddings
373
+ self.rope_theta = config.rope_theta
374
+ self.is_causal = True
375
+
376
+ if (self.head_dim * self.num_heads) != self.hidden_size:
377
+ raise ValueError(
378
+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
379
+ f" and `num_heads`: {self.num_heads})."
380
+ )
381
+
382
+ self.q_proj = nn.Linear(
383
+ self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias
384
+ )
385
+ self.k_proj = nn.Linear(
386
+ self.hidden_size,
387
+ self.num_key_value_heads * self.head_dim,
388
+ bias=config.attention_bias,
389
+ )
390
+ self.v_proj = nn.Linear(
391
+ self.hidden_size,
392
+ self.num_key_value_heads * self.head_dim,
393
+ bias=config.attention_bias,
394
+ )
395
+ self.o_proj = nn.Linear(
396
+ self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias
397
+ )
398
+ self._init_rope()
399
+
400
+ def _init_rope(self):
401
+ if self.config.rope_scaling is None:
402
+ self.rotary_emb = MiniCPMRotaryEmbedding(
403
+ self.head_dim,
404
+ max_position_embeddings=self.max_position_embeddings,
405
+ base=self.rope_theta,
406
+ )
407
+ else:
408
+ scaling_type = self.config.rope_scaling["type"]
409
+ scaling_factor = self.config.rope_scaling["factor"]
410
+ if scaling_type == "linear":
411
+ self.rotary_emb = MiniCPMLinearScalingRotaryEmbedding(
412
+ self.head_dim,
413
+ max_position_embeddings=self.max_position_embeddings,
414
+ scaling_factor=scaling_factor,
415
+ base=self.rope_theta,
416
+ )
417
+ elif scaling_type == "dynamic":
418
+ self.rotary_emb = MiniCPMDynamicNTKScalingRotaryEmbedding(
419
+ self.head_dim,
420
+ max_position_embeddings=self.max_position_embeddings,
421
+ scaling_factor=scaling_factor,
422
+ base=self.rope_theta,
423
+ )
424
+ else:
425
+ raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
426
+
427
+ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
428
+ return (
429
+ tensor.view(bsz, seq_len, self.num_heads, self.head_dim)
430
+ .transpose(1, 2)
431
+ .contiguous()
432
+ )
433
+
434
+ def forward(
435
+ self,
436
+ hidden_states: torch.Tensor,
437
+ attention_mask: Optional[torch.Tensor] = None,
438
+ position_ids: Optional[torch.LongTensor] = None,
439
+ past_key_value: Optional[Cache] = None,
440
+ output_attentions: bool = False,
441
+ use_cache: bool = False,
442
+ **kwargs,
443
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
444
+ if "padding_mask" in kwargs:
445
+ warnings.warn(
446
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
447
+ )
448
+
449
+ bsz, q_len, _ = hidden_states.size()
450
+
451
+ if self.config.pretraining_tp > 1:
452
+ key_value_slicing = (
453
+ self.num_key_value_heads * self.head_dim
454
+ ) // self.config.pretraining_tp
455
+ query_slices = self.q_proj.weight.split(
456
+ (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0
457
+ )
458
+ key_slices = self.k_proj.weight.split(key_value_slicing, dim=0)
459
+ value_slices = self.v_proj.weight.split(key_value_slicing, dim=0)
460
+
461
+ query_states = [
462
+ F.linear(hidden_states, query_slices[i])
463
+ for i in range(self.config.pretraining_tp)
464
+ ]
465
+ query_states = torch.cat(query_states, dim=-1)
466
+
467
+ key_states = [
468
+ F.linear(hidden_states, key_slices[i])
469
+ for i in range(self.config.pretraining_tp)
470
+ ]
471
+ key_states = torch.cat(key_states, dim=-1)
472
+
473
+ value_states = [
474
+ F.linear(hidden_states, value_slices[i])
475
+ for i in range(self.config.pretraining_tp)
476
+ ]
477
+ value_states = torch.cat(value_states, dim=-1)
478
+
479
+ else:
480
+ query_states = self.q_proj(hidden_states)
481
+ key_states = self.k_proj(hidden_states)
482
+ value_states = self.v_proj(hidden_states)
483
+
484
+ query_states = query_states.view(
485
+ bsz, q_len, self.num_heads, self.head_dim
486
+ ).transpose(1, 2)
487
+ key_states = key_states.view(
488
+ bsz, q_len, self.num_key_value_heads, self.head_dim
489
+ ).transpose(1, 2)
490
+ value_states = value_states.view(
491
+ bsz, q_len, self.num_key_value_heads, self.head_dim
492
+ ).transpose(1, 2)
493
+
494
+ kv_seq_len = key_states.shape[-2]
495
+ if past_key_value is not None:
496
+ if self.layer_idx is None:
497
+ raise ValueError(
498
+ f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
499
+ "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
500
+ "with a layer index."
501
+ )
502
+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
503
+ cos, sin = self.rotary_emb(value_states.to(torch.float32), seq_len=kv_seq_len)
504
+
505
+ query_states, key_states = apply_rotary_pos_emb(
506
+ query_states, key_states, cos, sin, position_ids
507
+ )
508
+
509
+ if past_key_value is not None:
510
+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
511
+ key_states, value_states = past_key_value.update(
512
+ key_states, value_states, self.layer_idx, cache_kwargs
513
+ )
514
+
515
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
516
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
517
+
518
+ attn_weights = torch.matmul(
519
+ query_states, key_states.transpose(2, 3)
520
+ ) / math.sqrt(self.head_dim)
521
+ if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
522
+ raise ValueError(
523
+ f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
524
+ f" {attn_weights.size()}"
525
+ )
526
+
527
+ if attention_mask is not None:
528
+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
529
+ raise ValueError(
530
+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
531
+ )
532
+ attn_weights = attn_weights + attention_mask
533
+
534
+ # upcast attention to fp32
535
+ attn_weights = nn.functional.softmax(
536
+ attn_weights, dim=-1, dtype=torch.float32
537
+ ).to(query_states.dtype)
538
+ attn_weights = nn.functional.dropout(
539
+ attn_weights, p=self.attention_dropout, training=self.training
540
+ )
541
+ attn_output = torch.matmul(attn_weights, value_states)
542
+
543
+ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
544
+ raise ValueError(
545
+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
546
+ f" {attn_output.size()}"
547
+ )
548
+
549
+ attn_output = attn_output.transpose(1, 2).contiguous()
550
+
551
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
552
+
553
+ if self.config.pretraining_tp > 1:
554
+ attn_output = attn_output.split(
555
+ self.hidden_size // self.config.pretraining_tp, dim=2
556
+ )
557
+ o_proj_slices = self.o_proj.weight.split(
558
+ self.hidden_size // self.config.pretraining_tp, dim=1
559
+ )
560
+ attn_output = sum(
561
+ [
562
+ F.linear(attn_output[i], o_proj_slices[i])
563
+ for i in range(self.config.pretraining_tp)
564
+ ]
565
+ )
566
+ else:
567
+ attn_output = self.o_proj(attn_output)
568
+
569
+ if not output_attentions:
570
+ attn_weights = None
571
+
572
+ return attn_output, attn_weights, past_key_value
573
+
574
+
575
+ class MiniCPMFlashAttention2(MiniCPMAttention):
576
+ """
577
+ MiniCPM flash attention module. This module inherits from `MiniCPMAttention` as the weights of the module stays
578
+ untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
579
+ flash attention and deal with padding tokens in case the input contains any of them.
580
+ """
581
+
582
+ def __init__(self, *args, **kwargs):
583
+ super().__init__(*args, **kwargs)
584
+
585
+ # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
586
+ # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
587
+ # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
588
+ self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
589
+
590
+ def forward(
591
+ self,
592
+ hidden_states: torch.Tensor,
593
+ attention_mask: Optional[torch.LongTensor] = None,
594
+ position_ids: Optional[torch.LongTensor] = None,
595
+ past_key_value: Optional[Cache] = None,
596
+ output_attentions: bool = False,
597
+ use_cache: bool = False,
598
+ **kwargs,
599
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
600
+ # MiniCPMFlashAttention2 attention does not support output_attentions
601
+ if "padding_mask" in kwargs:
602
+ warnings.warn(
603
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
604
+ )
605
+
606
+ # overwrite attention_mask with padding_mask
607
+ attention_mask = kwargs.pop("padding_mask")
608
+
609
+ output_attentions = False
610
+
611
+ bsz, q_len, _ = hidden_states.size()
612
+
613
+ query_states = self.q_proj(hidden_states)
614
+ key_states = self.k_proj(hidden_states)
615
+ value_states = self.v_proj(hidden_states)
616
+
617
+ # Flash attention requires the input to have the shape
618
+ # batch_size x seq_length x head_dim x hidden_dim
619
+ # therefore we just need to keep the original shape
620
+ query_states = query_states.view(
621
+ bsz, q_len, self.num_heads, self.head_dim
622
+ ).transpose(1, 2)
623
+ key_states = key_states.view(
624
+ bsz, q_len, self.num_key_value_heads, self.head_dim
625
+ ).transpose(1, 2)
626
+ value_states = value_states.view(
627
+ bsz, q_len, self.num_key_value_heads, self.head_dim
628
+ ).transpose(1, 2)
629
+
630
+ kv_seq_len = key_states.shape[-2]
631
+ if past_key_value is not None:
632
+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
633
+ cos, sin = self.rotary_emb(value_states.to(torch.float32), seq_len=kv_seq_len)
634
+ query_states, key_states = apply_rotary_pos_emb(
635
+ query_states, key_states, cos, sin, position_ids
636
+ )
637
+
638
+ if past_key_value is not None:
639
+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
640
+ key_states, value_states = past_key_value.update(
641
+ key_states, value_states, self.layer_idx, cache_kwargs
642
+ )
643
+
644
+ # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
645
+ # to be able to avoid many of these transpose/reshape/view.
646
+ query_states = query_states.transpose(1, 2)
647
+ key_states = key_states.transpose(1, 2)
648
+ value_states = value_states.transpose(1, 2)
649
+
650
+ dropout_rate = self.attention_dropout if self.training else 0.0
651
+
652
+ # In PEFT, usually we cast the layer norms in float32 for training stability reasons
653
+ # therefore the input hidden states gets silently casted in float32. Hence, we need
654
+ # cast them back in the correct dtype just to be sure everything works as expected.
655
+ # This might slowdown training & inference so it is recommended to not cast the LayerNorms
656
+ # in fp32. (MiniCPMRMSNorm handles it correctly)
657
+
658
+ input_dtype = query_states.dtype
659
+ if input_dtype == torch.float32:
660
+ # Handle the case where the model is quantized
661
+ if hasattr(self.config, "_pre_quantization_dtype"):
662
+ target_dtype = self.config._pre_quantization_dtype
663
+ else:
664
+ target_dtype = self.q_proj.weight.dtype
665
+
666
+ logger.warning_once(
667
+ f"The input hidden states seems to be silently casted in float32, this might be related to"
668
+ f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
669
+ f" {target_dtype}."
670
+ )
671
+
672
+ query_states = query_states.to(target_dtype)
673
+ key_states = key_states.to(target_dtype)
674
+ value_states = value_states.to(target_dtype)
675
+
676
+ attn_output = self._flash_attention_forward(
677
+ query_states,
678
+ key_states,
679
+ value_states,
680
+ attention_mask,
681
+ q_len,
682
+ dropout=dropout_rate,
683
+ )
684
+
685
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
686
+ attn_output = self.o_proj(attn_output)
687
+
688
+ if not output_attentions:
689
+ attn_weights = None
690
+
691
+ return attn_output, attn_weights, past_key_value
692
+
693
+ def _flash_attention_forward(
694
+ self,
695
+ query_states,
696
+ key_states,
697
+ value_states,
698
+ attention_mask,
699
+ query_length,
700
+ dropout=0.0,
701
+ softmax_scale=None,
702
+ ):
703
+ """
704
+ Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
705
+ first unpad the input, then computes the attention scores and pad the final attention scores.
706
+ Args:
707
+ query_states (`torch.Tensor`):
708
+ Input query states to be passed to Flash Attention API
709
+ key_states (`torch.Tensor`):
710
+ Input key states to be passed to Flash Attention API
711
+ value_states (`torch.Tensor`):
712
+ Input value states to be passed to Flash Attention API
713
+ attention_mask (`torch.Tensor`):
714
+ The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
715
+ position of padding tokens and 1 for the position of non-padding tokens.
716
+ dropout (`int`, *optional*):
717
+ Attention dropout
718
+ softmax_scale (`float`, *optional*):
719
+ The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
720
+ """
721
+ if not self._flash_attn_uses_top_left_mask:
722
+ causal = self.is_causal
723
+ else:
724
+ # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in MiniCPMFlashAttention2 __init__.
725
+ causal = self.is_causal and query_length != 1
726
+ # Contains at least one padding token in the sequence
727
+ if attention_mask is not None:
728
+ batch_size = query_states.shape[0]
729
+ (
730
+ query_states,
731
+ key_states,
732
+ value_states,
733
+ indices_q,
734
+ cu_seq_lens,
735
+ max_seq_lens,
736
+ ) = self._upad_input(
737
+ query_states, key_states, value_states, attention_mask, query_length
738
+ )
739
+
740
+ cu_seqlens_q, cu_seqlens_k = cu_seq_lens
741
+ max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
742
+ attn_output_unpad = flash_attn_varlen_func(
743
+ query_states,
744
+ key_states,
745
+ value_states,
746
+ cu_seqlens_q=cu_seqlens_q,
747
+ cu_seqlens_k=cu_seqlens_k,
748
+ max_seqlen_q=max_seqlen_in_batch_q,
749
+ max_seqlen_k=max_seqlen_in_batch_k,
750
+ dropout_p=dropout,
751
+ softmax_scale=softmax_scale,
752
+ causal=causal,
753
+ )
754
+
755
+ attn_output = pad_input(
756
+ attn_output_unpad, indices_q, batch_size, query_length
757
+ )
758
+ else:
759
+ attn_output = flash_attn_func(
760
+ query_states,
761
+ key_states,
762
+ value_states,
763
+ dropout,
764
+ softmax_scale=softmax_scale,
765
+ causal=causal,
766
+ )
767
+
768
+ return attn_output
769
+
770
+ def _upad_input(
771
+ self, query_layer, key_layer, value_layer, attention_mask, query_length
772
+ ):
773
+ indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
774
+ batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
775
+
776
+ key_layer = index_first_axis(
777
+ key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
778
+ indices_k,
779
+ )
780
+ value_layer = index_first_axis(
781
+ value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
782
+ indices_k,
783
+ )
784
+ if query_length == kv_seq_len:
785
+ query_layer = index_first_axis(
786
+ query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim),
787
+ indices_k,
788
+ )
789
+ cu_seqlens_q = cu_seqlens_k
790
+ max_seqlen_in_batch_q = max_seqlen_in_batch_k
791
+ indices_q = indices_k
792
+ elif query_length == 1:
793
+ max_seqlen_in_batch_q = 1
794
+ cu_seqlens_q = torch.arange(
795
+ batch_size + 1, dtype=torch.int32, device=query_layer.device
796
+ ) # There is a memcpy here, that is very bad.
797
+ indices_q = cu_seqlens_q[:-1]
798
+ query_layer = query_layer.squeeze(1)
799
+ else:
800
+ # The -q_len: slice assumes left padding.
801
+ attention_mask = attention_mask[:, -query_length:]
802
+ query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(
803
+ query_layer, attention_mask
804
+ )
805
+
806
+ return (
807
+ query_layer,
808
+ key_layer,
809
+ value_layer,
810
+ indices_q,
811
+ (cu_seqlens_q, cu_seqlens_k),
812
+ (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
813
+ )
814
+
815
+
816
+ class MiniCPMSdpaAttention(MiniCPMAttention):
817
+ """
818
+ MiniCPM attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
819
+ `MiniCPMAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
820
+ SDPA API.
821
+ """
822
+
823
+ # Adapted from MiniCPMAttention.forward
824
+ def forward(
825
+ self,
826
+ hidden_states: torch.Tensor,
827
+ attention_mask: Optional[torch.Tensor] = None,
828
+ position_ids: Optional[torch.LongTensor] = None,
829
+ past_key_value: Optional[Cache] = None,
830
+ output_attentions: bool = False,
831
+ use_cache: bool = False,
832
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
833
+ if output_attentions:
834
+ # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
835
+ logger.warning_once(
836
+ "MiniCPMModel is using MiniCPMSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
837
+ 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
838
+ )
839
+ return super().forward(
840
+ hidden_states=hidden_states,
841
+ attention_mask=attention_mask,
842
+ position_ids=position_ids,
843
+ past_key_value=past_key_value,
844
+ output_attentions=output_attentions,
845
+ use_cache=use_cache,
846
+ )
847
+
848
+ bsz, q_len, _ = hidden_states.size()
849
+
850
+ query_states = self.q_proj(hidden_states)
851
+ key_states = self.k_proj(hidden_states)
852
+ value_states = self.v_proj(hidden_states)
853
+
854
+ query_states = query_states.view(
855
+ bsz, q_len, self.num_heads, self.head_dim
856
+ ).transpose(1, 2)
857
+ key_states = key_states.view(
858
+ bsz, q_len, self.num_key_value_heads, self.head_dim
859
+ ).transpose(1, 2)
860
+ value_states = value_states.view(
861
+ bsz, q_len, self.num_key_value_heads, self.head_dim
862
+ ).transpose(1, 2)
863
+
864
+ kv_seq_len = key_states.shape[-2]
865
+ if past_key_value is not None:
866
+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
867
+ cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
868
+
869
+ query_states, key_states = apply_rotary_pos_emb(
870
+ query_states, key_states, cos, sin, position_ids
871
+ )
872
+
873
+ if past_key_value is not None:
874
+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
875
+ key_states, value_states = past_key_value.update(
876
+ key_states, value_states, self.layer_idx, cache_kwargs
877
+ )
878
+
879
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
880
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
881
+
882
+ if attention_mask is not None:
883
+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
884
+ raise ValueError(
885
+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
886
+ )
887
+
888
+ # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
889
+ # Reference: https://github.com/pytorch/pytorch/issues/112577.
890
+ if query_states.device.type == "cuda" and attention_mask is not None:
891
+ query_states = query_states.contiguous()
892
+ key_states = key_states.contiguous()
893
+ value_states = value_states.contiguous()
894
+
895
+ attn_output = torch.nn.functional.scaled_dot_product_attention(
896
+ query_states,
897
+ key_states,
898
+ value_states,
899
+ attn_mask=attention_mask,
900
+ dropout_p=self.attention_dropout if self.training else 0.0,
901
+ # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.
902
+ is_causal=self.is_causal and attention_mask is None and q_len > 1,
903
+ )
904
+
905
+ attn_output = attn_output.transpose(1, 2).contiguous()
906
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
907
+
908
+ attn_output = self.o_proj(attn_output)
909
+
910
+ return attn_output, None, past_key_value
911
+
912
+
913
+ MINICPM_ATTENTION_CLASSES = {
914
+ "eager": MiniCPMAttention,
915
+ "flash_attention_2": MiniCPMFlashAttention2,
916
+ "sdpa": MiniCPMSdpaAttention,
917
+ }
918
+
919
+
920
+ class MiniCPMDecoderLayer(nn.Module):
921
+ def __init__(self, config: MiniCPMConfig, layer_idx: int):
922
+ super().__init__()
923
+ self.hidden_size = config.hidden_size
924
+ self.self_attn = MINICPM_ATTENTION_CLASSES[config._attn_implementation](
925
+ config=config, layer_idx=layer_idx
926
+ )
927
+
928
+ self.mlp = MiniCPMMLP(config)
929
+ self.input_layernorm = MiniCPMRMSNorm(
930
+ config.hidden_size, eps=config.rms_norm_eps
931
+ )
932
+ self.post_attention_layernorm = MiniCPMRMSNorm(
933
+ config.hidden_size, eps=config.rms_norm_eps
934
+ )
935
+
936
+ self.scale_depth = config.scale_depth
937
+ self.num_hidden_layers = config.num_hidden_layers
938
+
939
+ def forward(
940
+ self,
941
+ hidden_states: torch.Tensor,
942
+ attention_mask: Optional[torch.Tensor] = None,
943
+ position_ids: Optional[torch.LongTensor] = None,
944
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
945
+ output_attentions: Optional[bool] = False,
946
+ use_cache: Optional[bool] = False,
947
+ **kwargs,
948
+ ) -> Tuple[
949
+ torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
950
+ ]:
951
+ """
952
+ Args:
953
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
954
+ attention_mask (`torch.FloatTensor`, *optional*):
955
+ attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
956
+ query_sequence_length, key_sequence_length)` if default attention is used.
957
+ output_attentions (`bool`, *optional*):
958
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
959
+ returned tensors for more detail.
960
+ use_cache (`bool`, *optional*):
961
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
962
+ (see `past_key_values`).
963
+ past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
964
+ """
965
+ if "padding_mask" in kwargs:
966
+ warnings.warn(
967
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
968
+ )
969
+
970
+ residual = hidden_states
971
+ hidden_states = self.input_layernorm(hidden_states)
972
+ # Self Attention
973
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
974
+ hidden_states=hidden_states,
975
+ attention_mask=attention_mask,
976
+ position_ids=position_ids,
977
+ past_key_value=past_key_value,
978
+ output_attentions=output_attentions,
979
+ use_cache=use_cache,
980
+ **kwargs,
981
+ )
982
+
983
+ hidden_states = residual + hidden_states * (
984
+ self.scale_depth / math.sqrt(self.num_hidden_layers)
985
+ )
986
+
987
+ # Fully Connected
988
+ residual = hidden_states
989
+ hidden_states = self.post_attention_layernorm(hidden_states)
990
+
991
+ hidden_states = self.mlp(hidden_states)
992
+ hidden_states = residual + hidden_states * (
993
+ self.scale_depth / math.sqrt(self.num_hidden_layers)
994
+ )
995
+
996
+ outputs = (hidden_states,)
997
+
998
+ if output_attentions:
999
+ outputs += (self_attn_weights,)
1000
+
1001
+ if use_cache:
1002
+ outputs += (present_key_value,)
1003
+
1004
+ return outputs
1005
+
1006
+
1007
+ MINICPM_START_DOCSTRING = r"""
1008
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
1009
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
1010
+ etc.)
1011
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
1012
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
1013
+ and behavior.
1014
+ Parameters:
1015
+ config ([`MiniCPMConfig`]):
1016
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
1017
+ load the weights associated with the model, only the configuration. Check out the
1018
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
1019
+ """
1020
+
1021
+
1022
+ @add_start_docstrings(
1023
+ "The bare MiniCPM Model outputting raw hidden-states without any specific head on top.",
1024
+ MINICPM_START_DOCSTRING,
1025
+ )
1026
+ class MiniCPMPreTrainedModel(PreTrainedModel):
1027
+ config_class = MiniCPMConfig
1028
+ base_model_prefix = "model"
1029
+ supports_gradient_checkpointing = True
1030
+ _no_split_modules = ["MiniCPMDecoderLayer"]
1031
+ _skip_keys_device_placement = "past_key_values"
1032
+ _supports_flash_attn_2 = True
1033
+ _supports_sdpa = True
1034
+ _supports_cache_class = True
1035
+
1036
+ def _init_weights(self, module):
1037
+ std = self.config.initializer_range
1038
+ if isinstance(module, nn.Linear):
1039
+ module.weight.data.normal_(mean=0.0, std=std)
1040
+ if module.bias is not None:
1041
+ module.bias.data.zero_()
1042
+ elif isinstance(module, nn.Embedding):
1043
+ module.weight.data.normal_(mean=0.0, std=std)
1044
+ if module.padding_idx is not None:
1045
+ module.weight.data[module.padding_idx].zero_()
1046
+
1047
+
1048
+ MINICPM_INPUTS_DOCSTRING = r"""
1049
+ Args:
1050
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
1051
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
1052
+ it.
1053
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
1054
+ [`PreTrainedTokenizer.__call__`] for details.
1055
+ [What are input IDs?](../glossary#input-ids)
1056
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
1057
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
1058
+ - 1 for tokens that are **not masked**,
1059
+ - 0 for tokens that are **masked**.
1060
+ [What are attention masks?](../glossary#attention-mask)
1061
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
1062
+ [`PreTrainedTokenizer.__call__`] for details.
1063
+ If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
1064
+ `past_key_values`).
1065
+ If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
1066
+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
1067
+ information on the default strategy.
1068
+ - 1 indicates the head is **not masked**,
1069
+ - 0 indicates the head is **masked**.
1070
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1071
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
1072
+ config.n_positions - 1]`.
1073
+ [What are position IDs?](../glossary#position-ids)
1074
+ past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
1075
+ Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
1076
+ blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
1077
+ returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
1078
+ Two formats are allowed:
1079
+ - a [`~cache_utils.Cache`] instance;
1080
+ - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
1081
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
1082
+ cache format.
1083
+ The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
1084
+ legacy cache format will be returned.
1085
+ If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
1086
+ have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
1087
+ of shape `(batch_size, sequence_length)`.
1088
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
1089
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
1090
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
1091
+ model's internal embedding lookup matrix.
1092
+ use_cache (`bool`, *optional*):
1093
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
1094
+ `past_key_values`).
1095
+ output_attentions (`bool`, *optional*):
1096
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
1097
+ tensors for more detail.
1098
+ output_hidden_states (`bool`, *optional*):
1099
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
1100
+ more detail.
1101
+ return_dict (`bool`, *optional*):
1102
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
1103
+ """
1104
+
1105
+
1106
+ @add_start_docstrings(
1107
+ "The bare MiniCPM Model outputting raw hidden-states without any specific head on top.",
1108
+ MINICPM_START_DOCSTRING,
1109
+ )
1110
+ class MiniCPMModel(MiniCPMPreTrainedModel):
1111
+ """
1112
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MiniCPMDecoderLayer`]
1113
+ Args:
1114
+ config: MiniCPMConfig
1115
+ """
1116
+
1117
+ def __init__(self, config: MiniCPMConfig):
1118
+ super().__init__(config)
1119
+ self.padding_idx = config.pad_token_id
1120
+ self.vocab_size = config.vocab_size
1121
+
1122
+ self.embed_tokens = nn.Embedding(
1123
+ config.vocab_size, config.hidden_size, self.padding_idx
1124
+ )
1125
+ self.layers = nn.ModuleList(
1126
+ [
1127
+ MiniCPMDecoderLayer(config, layer_idx)
1128
+ for layer_idx in range(config.num_hidden_layers)
1129
+ ]
1130
+ )
1131
+ self._use_sdpa = config._attn_implementation == "sdpa"
1132
+ self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
1133
+
1134
+ self.norm = MiniCPMRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
1135
+
1136
+ self.gradient_checkpointing = False
1137
+ # Initialize weights and apply final processing
1138
+ self.post_init()
1139
+
1140
+ def get_input_embeddings(self):
1141
+ return self.embed_tokens
1142
+
1143
+ def set_input_embeddings(self, value):
1144
+ self.embed_tokens = value
1145
+
1146
+ @add_start_docstrings_to_model_forward(MINICPM_INPUTS_DOCSTRING)
1147
+ def forward(
1148
+ self,
1149
+ input_ids: torch.LongTensor = None,
1150
+ attention_mask: Optional[torch.Tensor] = None,
1151
+ position_ids: Optional[torch.LongTensor] = None,
1152
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
1153
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1154
+ use_cache: Optional[bool] = None,
1155
+ output_attentions: Optional[bool] = None,
1156
+ output_hidden_states: Optional[bool] = None,
1157
+ return_dict: Optional[bool] = None,
1158
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
1159
+ output_attentions = (
1160
+ output_attentions
1161
+ if output_attentions is not None
1162
+ else self.config.output_attentions
1163
+ )
1164
+ output_hidden_states = (
1165
+ output_hidden_states
1166
+ if output_hidden_states is not None
1167
+ else self.config.output_hidden_states
1168
+ )
1169
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
1170
+
1171
+ return_dict = (
1172
+ return_dict if return_dict is not None else self.config.use_return_dict
1173
+ )
1174
+
1175
+ # retrieve input_ids and inputs_embeds
1176
+ if input_ids is not None and inputs_embeds is not None:
1177
+ raise ValueError(
1178
+ "You cannot specify both input_ids and inputs_embeds at the same time"
1179
+ )
1180
+ elif input_ids is not None:
1181
+ batch_size, seq_length = input_ids.shape[:2]
1182
+ elif inputs_embeds is not None:
1183
+ batch_size, seq_length = inputs_embeds.shape[:2]
1184
+ else:
1185
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
1186
+
1187
+ if self.gradient_checkpointing and self.training:
1188
+ if use_cache:
1189
+ logger.warning_once(
1190
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
1191
+ )
1192
+ use_cache = False
1193
+
1194
+ past_key_values_length = 0
1195
+ if use_cache:
1196
+ use_legacy_cache = not isinstance(past_key_values, Cache)
1197
+ if use_legacy_cache:
1198
+ past_key_values = DynamicCache.from_legacy_cache(past_key_values)
1199
+ past_key_values_length = past_key_values.get_usable_length(seq_length)
1200
+
1201
+ if position_ids is None:
1202
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
1203
+ position_ids = torch.arange(
1204
+ past_key_values_length,
1205
+ seq_length + past_key_values_length,
1206
+ dtype=torch.long,
1207
+ device=device,
1208
+ )
1209
+ position_ids = position_ids.unsqueeze(0)
1210
+
1211
+ if inputs_embeds is None:
1212
+ inputs_embeds = self.embed_tokens(input_ids) * self.config.scale_emb
1213
+
1214
+ if self._use_flash_attention_2:
1215
+ # 2d mask is passed through the layers
1216
+ attention_mask = (
1217
+ attention_mask
1218
+ if (attention_mask is not None and 0 in attention_mask)
1219
+ else None
1220
+ )
1221
+ elif self._use_sdpa and not output_attentions:
1222
+ # output_attentions=True can not be supported when using SDPA, and we fall back on
1223
+ # the manual implementation that requires a 4D causal mask in all cases.
1224
+ attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
1225
+ attention_mask,
1226
+ (batch_size, seq_length),
1227
+ inputs_embeds,
1228
+ past_key_values_length,
1229
+ )
1230
+ else:
1231
+ # 4d mask is passed through the layers
1232
+ attention_mask = _prepare_4d_causal_attention_mask(
1233
+ attention_mask,
1234
+ (batch_size, seq_length),
1235
+ inputs_embeds,
1236
+ past_key_values_length,
1237
+ )
1238
+
1239
+ # embed positions
1240
+ hidden_states = inputs_embeds
1241
+
1242
+ # decoder layers
1243
+ all_hidden_states = () if output_hidden_states else None
1244
+ all_self_attns = () if output_attentions else None
1245
+ next_decoder_cache = None
1246
+
1247
+ for decoder_layer in self.layers:
1248
+ if output_hidden_states:
1249
+ all_hidden_states += (hidden_states,)
1250
+
1251
+ if self.gradient_checkpointing and self.training:
1252
+ layer_outputs = self._gradient_checkpointing_func(
1253
+ decoder_layer.__call__,
1254
+ hidden_states,
1255
+ attention_mask,
1256
+ position_ids,
1257
+ past_key_values,
1258
+ output_attentions,
1259
+ use_cache,
1260
+ )
1261
+ else:
1262
+ layer_outputs = decoder_layer(
1263
+ hidden_states,
1264
+ attention_mask=attention_mask,
1265
+ position_ids=position_ids,
1266
+ past_key_value=past_key_values,
1267
+ output_attentions=output_attentions,
1268
+ use_cache=use_cache,
1269
+ )
1270
+
1271
+ hidden_states = layer_outputs[0]
1272
+
1273
+ if use_cache:
1274
+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]
1275
+
1276
+ if output_attentions:
1277
+ all_self_attns += (layer_outputs[1],)
1278
+
1279
+ hidden_states = self.norm(hidden_states)
1280
+
1281
+ # add hidden states from the last decoder layer
1282
+ if output_hidden_states:
1283
+ all_hidden_states += (hidden_states,)
1284
+
1285
+ next_cache = None
1286
+ if use_cache:
1287
+ next_cache = (
1288
+ next_decoder_cache.to_legacy_cache()
1289
+ if use_legacy_cache
1290
+ else next_decoder_cache
1291
+ )
1292
+ if not return_dict:
1293
+ return tuple(
1294
+ v
1295
+ for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
1296
+ if v is not None
1297
+ )
1298
+ return BaseModelOutputWithPast(
1299
+ last_hidden_state=hidden_states,
1300
+ past_key_values=next_cache,
1301
+ hidden_states=all_hidden_states,
1302
+ attentions=all_self_attns,
1303
+ )
1304
+
1305
+
1306
+ class MiniCPMForCausalLM(MiniCPMPreTrainedModel):
1307
+ _tied_weights_keys = ["lm_head.weight"]
1308
+
1309
+ def __init__(self, config):
1310
+ super().__init__(config)
1311
+ self.model = MiniCPMModel(config)
1312
+ self.vocab_size = config.vocab_size
1313
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
1314
+
1315
+ # Initialize weights and apply final processing
1316
+ self.post_init()
1317
+
1318
+ def get_input_embeddings(self):
1319
+ return self.model.embed_tokens
1320
+
1321
+ def set_input_embeddings(self, value):
1322
+ self.model.embed_tokens = value
1323
+
1324
+ def get_output_embeddings(self):
1325
+ return self.lm_head
1326
+
1327
+ def set_output_embeddings(self, new_embeddings):
1328
+ self.lm_head = new_embeddings
1329
+
1330
+ def set_decoder(self, decoder):
1331
+ self.model = decoder
1332
+
1333
+ def get_decoder(self):
1334
+ return self.model
1335
+
1336
+ @add_start_docstrings_to_model_forward(MINICPM_INPUTS_DOCSTRING)
1337
+ @replace_return_docstrings(
1338
+ output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC
1339
+ )
1340
+ def forward(
1341
+ self,
1342
+ input_ids: torch.LongTensor = None,
1343
+ attention_mask: Optional[torch.Tensor] = None,
1344
+ position_ids: Optional[torch.LongTensor] = None,
1345
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
1346
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1347
+ labels: Optional[torch.LongTensor] = None,
1348
+ use_cache: Optional[bool] = None,
1349
+ output_attentions: Optional[bool] = None,
1350
+ output_hidden_states: Optional[bool] = None,
1351
+ return_dict: Optional[bool] = None,
1352
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1353
+ r"""
1354
+ Args:
1355
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1356
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
1357
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
1358
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
1359
+ Returns:
1360
+ Example:
1361
+ ```python
1362
+ >>> from transformers import AutoTokenizer, MiniCPMForCausalLM
1363
+ >>> model = MiniCPMForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
1364
+ >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
1365
+ >>> prompt = "Hey, are you conscious? Can you talk to me?"
1366
+ >>> inputs = tokenizer(prompt, return_tensors="pt")
1367
+ >>> # Generate
1368
+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
1369
+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
1370
+ "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
1371
+ ```"""
1372
+ output_attentions = (
1373
+ output_attentions
1374
+ if output_attentions is not None
1375
+ else self.config.output_attentions
1376
+ )
1377
+ output_hidden_states = (
1378
+ output_hidden_states
1379
+ if output_hidden_states is not None
1380
+ else self.config.output_hidden_states
1381
+ )
1382
+ return_dict = (
1383
+ return_dict if return_dict is not None else self.config.use_return_dict
1384
+ )
1385
+
1386
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
1387
+ outputs = self.model(
1388
+ input_ids=input_ids,
1389
+ attention_mask=attention_mask,
1390
+ position_ids=position_ids,
1391
+ past_key_values=past_key_values,
1392
+ inputs_embeds=inputs_embeds,
1393
+ use_cache=use_cache,
1394
+ output_attentions=output_attentions,
1395
+ output_hidden_states=output_hidden_states,
1396
+ return_dict=return_dict,
1397
+ )
1398
+
1399
+ hidden_states = outputs[0]
1400
+ if self.config.pretraining_tp > 1:
1401
+ lm_head_slices = self.lm_head.weight.split(
1402
+ self.vocab_size // self.config.pretraining_tp, dim=0
1403
+ )
1404
+ logits = [
1405
+ F.linear(hidden_states, lm_head_slices[i])
1406
+ for i in range(self.config.pretraining_tp)
1407
+ ]
1408
+ logits = torch.cat(logits, dim=-1)
1409
+ else:
1410
+ logits = self.lm_head(
1411
+ hidden_states / (self.config.hidden_size / self.config.dim_model_base)
1412
+ )
1413
+ logits = logits.float()
1414
+
1415
+ loss = None
1416
+ if labels is not None:
1417
+ # Shift so that tokens < n predict n
1418
+ shift_logits = logits[..., :-1, :].contiguous()
1419
+ shift_labels = labels[..., 1:].contiguous()
1420
+ # Flatten the tokens
1421
+ loss_fct = CrossEntropyLoss()
1422
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
1423
+ shift_labels = shift_labels.view(-1)
1424
+ # Enable model parallelism
1425
+ shift_labels = shift_labels.to(shift_logits.device)
1426
+ loss = loss_fct(shift_logits, shift_labels)
1427
+
1428
+ if not return_dict:
1429
+ output = (logits,) + outputs[1:]
1430
+ return (loss,) + output if loss is not None else output
1431
+
1432
+ return CausalLMOutputWithPast(
1433
+ loss=loss,
1434
+ logits=logits,
1435
+ past_key_values=outputs.past_key_values,
1436
+ hidden_states=outputs.hidden_states,
1437
+ attentions=outputs.attentions,
1438
+ )
1439
+
1440
+ def prepare_inputs_for_generation(
1441
+ self,
1442
+ input_ids,
1443
+ past_key_values=None,
1444
+ attention_mask=None,
1445
+ inputs_embeds=None,
1446
+ **kwargs,
1447
+ ):
1448
+ if past_key_values is not None:
1449
+ if isinstance(past_key_values, Cache):
1450
+ cache_length = past_key_values.get_seq_length()
1451
+ past_length = past_key_values.seen_tokens
1452
+ max_cache_length = past_key_values.get_max_length()
1453
+ else:
1454
+ cache_length = past_length = past_key_values[0][0].shape[2]
1455
+ max_cache_length = None
1456
+
1457
+ # Keep only the unprocessed tokens:
1458
+ # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
1459
+ # some of the inputs are exclusivelly passed as part of the cache (e.g. when passing input_embeds as
1460
+ # input)
1461
+ if (
1462
+ attention_mask is not None
1463
+ and attention_mask.shape[1] > input_ids.shape[1]
1464
+ ):
1465
+ input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
1466
+ # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
1467
+ # input_ids based on the past_length.
1468
+ elif past_length < input_ids.shape[1]:
1469
+ input_ids = input_ids[:, past_length:]
1470
+ # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
1471
+
1472
+ # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
1473
+ if (
1474
+ max_cache_length is not None
1475
+ and attention_mask is not None
1476
+ and cache_length + input_ids.shape[1] > max_cache_length
1477
+ ):
1478
+ attention_mask = attention_mask[:, -max_cache_length:]
1479
+
1480
+ position_ids = kwargs.get("position_ids", None)
1481
+ if attention_mask is not None and position_ids is None:
1482
+ # create position_ids on the fly for batch generation
1483
+ position_ids = attention_mask.long().cumsum(-1) - 1
1484
+ position_ids.masked_fill_(attention_mask == 0, 1)
1485
+ if past_key_values:
1486
+ position_ids = position_ids[:, -input_ids.shape[1] :]
1487
+
1488
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
1489
+ if inputs_embeds is not None and past_key_values is None:
1490
+ model_inputs = {"inputs_embeds": inputs_embeds}
1491
+ else:
1492
+ model_inputs = {"input_ids": input_ids}
1493
+
1494
+ model_inputs.update(
1495
+ {
1496
+ "position_ids": position_ids,
1497
+ "past_key_values": past_key_values,
1498
+ "use_cache": kwargs.get("use_cache"),
1499
+ "attention_mask": attention_mask,
1500
+ }
1501
+ )
1502
+ return model_inputs
1503
+
1504
+ @staticmethod
1505
+ def _reorder_cache(past_key_values, beam_idx):
1506
+ reordered_past = ()
1507
+ for layer_past in past_key_values:
1508
+ reordered_past += (
1509
+ tuple(
1510
+ past_state.index_select(0, beam_idx.to(past_state.device))
1511
+ for past_state in layer_past
1512
+ ),
1513
+ )
1514
+ return reordered_past
1515
+
1516
+ @torch.inference_mode()
1517
+ def chat(
1518
+ self,
1519
+ tokenizer,
1520
+ query: str,
1521
+ history: List[Dict] = None,
1522
+ role: str = "user",
1523
+ max_length: int = 4096,
1524
+ num_beams=1,
1525
+ do_sample=True,
1526
+ top_p=0.8,
1527
+ temperature=0.3,
1528
+ logits_processor=None,
1529
+ **kwargs,
1530
+ ):
1531
+ if history is None:
1532
+ history = []
1533
+ if logits_processor:
1534
+ gen_kwargs = {
1535
+ "max_length": max_length,
1536
+ "num_beams": num_beams,
1537
+ "do_sample": do_sample,
1538
+ "top_p": top_p,
1539
+ "temperature": temperature,
1540
+ "logits_processor": logits_processor,
1541
+ **kwargs,
1542
+ }
1543
+ else:
1544
+ gen_kwargs = {
1545
+ "max_length": max_length,
1546
+ "num_beams": num_beams,
1547
+ "do_sample": do_sample,
1548
+ "top_p": top_p,
1549
+ "temperature": temperature,
1550
+ "logits_processor": logits_processor,
1551
+ **kwargs,
1552
+ }
1553
+
1554
+ history.append({"role": role, "content": query})
1555
+ history_str = tokenizer.apply_chat_template(
1556
+ history, tokenize=False, add_generation_prompt=False
1557
+ )
1558
+ inputs = tokenizer(history_str, return_tensors="pt").to(self.device)
1559
+ outputs = self.generate(**inputs, **gen_kwargs)
1560
+ outputs = outputs.tolist()[0][len(inputs["input_ids"][0]) : -1]
1561
+ response = tokenizer.decode(outputs)
1562
+ pattern = re.compile(r".*?(?=<AI>|<用户>)", re.DOTALL)
1563
+ matches = pattern.findall(response)
1564
+ if len(matches) > 0:
1565
+ response = matches[0]
1566
+ history.append({"role": "assistant", "content": response})
1567
+ return response, history
1568
+
1569
+
1570
+ @add_start_docstrings(
1571
+ """
1572
+ The MiniCPM Model transformer with a sequence classification head on top (linear layer).
1573
+ [`MiniCPMForSequenceClassification`] uses the last token in order to do the classification, as other causal models
1574
+ (e.g. GPT-2) do.
1575
+ Since it does classification on the last token, it requires to know the position of the last token. If a
1576
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
1577
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
1578
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
1579
+ each row of the batch).
1580
+ """,
1581
+ MINICPM_START_DOCSTRING,
1582
+ )
1583
+ class MiniCPMForSequenceClassification(MiniCPMPreTrainedModel):
1584
+ def __init__(self, config):
1585
+ super().__init__(config)
1586
+ self.num_labels = config.num_labels
1587
+ self.model = MiniCPMModel(config)
1588
+ self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
1589
+
1590
+ # Initialize weights and apply final processing
1591
+ self.post_init()
1592
+
1593
+ def get_input_embeddings(self):
1594
+ return self.model.embed_tokens
1595
+
1596
+ def set_input_embeddings(self, value):
1597
+ self.model.embed_tokens = value
1598
+
1599
+ @add_start_docstrings_to_model_forward(MINICPM_INPUTS_DOCSTRING)
1600
+ def forward(
1601
+ self,
1602
+ input_ids: torch.LongTensor = None,
1603
+ attention_mask: Optional[torch.Tensor] = None,
1604
+ position_ids: Optional[torch.LongTensor] = None,
1605
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
1606
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1607
+ labels: Optional[torch.LongTensor] = None,
1608
+ use_cache: Optional[bool] = None,
1609
+ output_attentions: Optional[bool] = None,
1610
+ output_hidden_states: Optional[bool] = None,
1611
+ return_dict: Optional[bool] = None,
1612
+ ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
1613
+ r"""
1614
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1615
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1616
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1617
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1618
+ """
1619
+ return_dict = (
1620
+ return_dict if return_dict is not None else self.config.use_return_dict
1621
+ )
1622
+
1623
+ transformer_outputs = self.model(
1624
+ input_ids,
1625
+ attention_mask=attention_mask,
1626
+ position_ids=position_ids,
1627
+ past_key_values=past_key_values,
1628
+ inputs_embeds=inputs_embeds,
1629
+ use_cache=use_cache,
1630
+ output_attentions=output_attentions,
1631
+ output_hidden_states=output_hidden_states,
1632
+ return_dict=return_dict,
1633
+ )
1634
+ hidden_states = transformer_outputs[0]
1635
+ logits = self.score(hidden_states)
1636
+
1637
+ if input_ids is not None:
1638
+ batch_size = input_ids.shape[0]
1639
+ else:
1640
+ batch_size = inputs_embeds.shape[0]
1641
+
1642
+ if self.config.pad_token_id is None and batch_size != 1:
1643
+ raise ValueError(
1644
+ "Cannot handle batch sizes > 1 if no padding token is defined."
1645
+ )
1646
+ if self.config.pad_token_id is None:
1647
+ sequence_lengths = -1
1648
+ else:
1649
+ if input_ids is not None:
1650
+ sequence_lengths = (
1651
+ torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
1652
+ ).to(logits.device)
1653
+ else:
1654
+ sequence_lengths = -1
1655
+
1656
+ pooled_logits = logits[
1657
+ torch.arange(batch_size, device=logits.device), sequence_lengths
1658
+ ]
1659
+
1660
+ loss = None
1661
+ if labels is not None:
1662
+ labels = labels.to(logits.device)
1663
+ if self.config.problem_type is None:
1664
+ if self.num_labels == 1:
1665
+ self.config.problem_type = "regression"
1666
+ elif self.num_labels > 1 and (
1667
+ labels.dtype == torch.long or labels.dtype == torch.int
1668
+ ):
1669
+ self.config.problem_type = "single_label_classification"
1670
+ else:
1671
+ self.config.problem_type = "multi_label_classification"
1672
+
1673
+ if self.config.problem_type == "regression":
1674
+ loss_fct = MSELoss()
1675
+ if self.num_labels == 1:
1676
+ loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
1677
+ else:
1678
+ loss = loss_fct(pooled_logits, labels)
1679
+ elif self.config.problem_type == "single_label_classification":
1680
+ loss_fct = CrossEntropyLoss()
1681
+ loss = loss_fct(
1682
+ pooled_logits.view(-1, self.num_labels), labels.view(-1)
1683
+ )
1684
+ elif self.config.problem_type == "multi_label_classification":
1685
+ loss_fct = BCEWithLogitsLoss()
1686
+ loss = loss_fct(pooled_logits, labels)
1687
+ if not return_dict:
1688
+ output = (pooled_logits,) + transformer_outputs[1:]
1689
+ return ((loss,) + output) if loss is not None else output
1690
+
1691
+ return SequenceClassifierOutputWithPast(
1692
+ loss=loss,
1693
+ logits=pooled_logits,
1694
+ past_key_values=transformer_outputs.past_key_values,
1695
+ hidden_states=transformer_outputs.hidden_states,
1696
+ attentions=transformer_outputs.attentions,
1697
+ )
modeling_minicpmv.py ADDED
@@ -0,0 +1,575 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from typing import List, Optional
3
+ import json
4
+ import timm
5
+ import torch
6
+ import torchvision
7
+ from PIL import Image
8
+ from timm.data import IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
9
+ from torchvision import transforms
10
+ from transformers import LlamaTokenizer
11
+
12
+ from .configuration_minicpm import MiniCPMVConfig
13
+ from .modeling_minicpm import MiniCPMForCausalLM, MiniCPMPreTrainedModel
14
+ from .resampler import Resampler
15
+
16
+
17
+ class MiniCPMVPreTrainedModel(MiniCPMPreTrainedModel):
18
+ config_class = MiniCPMVConfig
19
+
20
+
21
+ class MiniCPMV(MiniCPMVPreTrainedModel):
22
+ def __init__(self, config):
23
+ super().__init__(config)
24
+
25
+ self.llm = MiniCPMForCausalLM(config)
26
+ self.vpm = self.init_vision_module()
27
+ self.vision_dim = self.vpm.embed_dim
28
+ self.embed_dim = self.llm.config.hidden_size
29
+ self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
30
+ self.transform = self.init_transform()
31
+
32
+ def init_vision_module(self):
33
+ model = timm.create_model(
34
+ self.config.vision_encoder,
35
+ pretrained=False,
36
+ num_classes=0,
37
+ dynamic_img_size=True,
38
+ dynamic_img_pad=True
39
+ )
40
+
41
+ if isinstance(model, timm.models.VisionTransformer):
42
+ if model.attn_pool is not None:
43
+ model.attn_pool = torch.nn.Identity()
44
+
45
+ if self.config.drop_vision_last_layer:
46
+ model.blocks = model.blocks[:-1]
47
+
48
+ return model
49
+
50
+ def init_resampler(self, embed_dim, vision_dim):
51
+ return Resampler(
52
+ grid_size=int(math.sqrt(self.config.query_num)),
53
+ embed_dim=embed_dim,
54
+ num_heads=embed_dim // 128,
55
+ kv_dim=vision_dim,
56
+ adaptive=True
57
+ )
58
+
59
+ def init_transform(self):
60
+ return transforms.Compose(
61
+ [
62
+ transforms.ToTensor(),
63
+ transforms.Normalize(
64
+ mean=IMAGENET_INCEPTION_MEAN, std=IMAGENET_INCEPTION_STD
65
+ ),
66
+ ]
67
+ )
68
+
69
+ def get_vision_embedding(self, pixel_values):
70
+ res = []
71
+ dtype = self.vpm.pos_embed.data.dtype
72
+ for pixel_value in pixel_values:
73
+ H, W = pixel_value.shape[-2:]
74
+ tgt_size = (
75
+ math.ceil(H / self.vpm.patch_embed.patch_size[0]), math.ceil(W / self.vpm.patch_embed.patch_size[0]))
76
+ vision_embedding = self.vpm.forward_features(pixel_value.unsqueeze(0).type(dtype))
77
+ if hasattr(self.vpm, 'num_prefix_tokens') and self.vpm.num_prefix_tokens > 0:
78
+ vision_embedding = vision_embedding[:, self.vpm.num_prefix_tokens:]
79
+ res.append(self.resampler(vision_embedding, tgt_size))
80
+ return torch.vstack(res)
81
+
82
+ def get_vllm_embedding(self, data):
83
+ if "vision_hidden_states" not in data:
84
+ pixel_values_list = data["pixel_values"]
85
+ vision_hidden_states = []
86
+ for pixel_values in pixel_values_list:
87
+ if len(pixel_values) > 0:
88
+ vision_hidden_states.append(self.get_vision_embedding(pixel_values))
89
+ elif self.training:
90
+ dtype = self.vpm.pos_embed.data.dtype
91
+ device = self.vpm.pos_embed.data.device
92
+ dummy_image = torch.zeros(
93
+ (1, 3, 224, 224), device=device, dtype=dtype
94
+ )
95
+ vision_hidden_states.append(self.get_vision_embedding(dummy_image))
96
+ else:
97
+ vision_hidden_states.append([])
98
+
99
+ else:
100
+ vision_hidden_states = data["vision_hidden_states"]
101
+
102
+ vllm_embedding = (
103
+ self.llm.model.embed_tokens(data["input_ids"]) * self.llm.config.scale_emb
104
+ )
105
+ vision_hidden_states = [
106
+ i.type(vllm_embedding.dtype) if isinstance(i, torch.Tensor) else i
107
+ for i in vision_hidden_states
108
+ ]
109
+
110
+ bs = len(data["input_ids"])
111
+ for i in range(bs):
112
+ cur_vs_hs = vision_hidden_states[i]
113
+ if len(cur_vs_hs) > 0:
114
+ cur_vllm_emb = vllm_embedding[i]
115
+ cur_image_bound = data["image_bound"][i]
116
+ if len(cur_image_bound) > 0:
117
+ image_indices = torch.stack(
118
+ [
119
+ torch.arange(r[0], r[1], dtype=torch.long)
120
+ for r in cur_image_bound
121
+ ]
122
+ ).to(vllm_embedding.device)
123
+
124
+ cur_vllm_emb.scatter_(
125
+ 0,
126
+ image_indices.view(-1, 1).repeat(1, cur_vllm_emb.shape[-1]),
127
+ cur_vs_hs.view(-1, cur_vs_hs.shape[-1]),
128
+ )
129
+ elif self.training:
130
+ cur_vllm_emb += cur_vs_hs[0].mean() * 0
131
+
132
+ return vllm_embedding, vision_hidden_states
133
+
134
+ def forward(self, data, **kwargs):
135
+ vllm_embedding, vision_hidden_states = self.get_vllm_embedding(data)
136
+ position_ids = data["position_ids"]
137
+ if position_ids.dtype != torch.int64:
138
+ position_ids = position_ids.long()
139
+
140
+ return self.llm(
141
+ input_ids=None,
142
+ position_ids=position_ids,
143
+ inputs_embeds=vllm_embedding,
144
+ **kwargs
145
+ )
146
+
147
+ def _convert_to_tensors(
148
+ self, tokenizer, input_str, max_inp_length: Optional[int] = None
149
+ ):
150
+ if tokenizer.add_bos_token:
151
+ input_ids = tokenizer.encode(input_str)
152
+ else:
153
+ input_ids = [tokenizer.bos_id] + tokenizer.encode(input_str)
154
+ if max_inp_length is not None:
155
+ input_ids = input_ids[:max_inp_length]
156
+ input_ids = torch.tensor(input_ids, dtype=torch.int32)
157
+
158
+ image_start_tokens = torch.where(input_ids == tokenizer.im_start_id)[0]
159
+ # 跳过 im_start
160
+ image_start_tokens += 1
161
+ image_end_tokens = torch.where(input_ids == tokenizer.im_end_id)[0]
162
+ valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
163
+ image_bound = torch.hstack(
164
+ [
165
+ image_start_tokens[:valid_image_nums].unsqueeze(-1),
166
+ image_end_tokens[:valid_image_nums].unsqueeze(-1),
167
+ ]
168
+ )
169
+
170
+ model_input = {}
171
+ model_input["input_ids"] = input_ids.unsqueeze(0).to(self.device)
172
+ model_input["image_bound"] = image_bound
173
+
174
+ return model_input
175
+
176
+ def _process_list(
177
+ self, tokenizer, data_list: List[str], max_inp_length: Optional[int] = None
178
+ ):
179
+ pad_keys = ["input_ids"]
180
+ input_tensors = []
181
+ for data in data_list:
182
+ input_tensors.append(
183
+ self._convert_to_tensors(tokenizer, data, max_inp_length)
184
+ )
185
+ padded = {}
186
+ for key in pad_keys:
187
+ padded[key] = pad(input_tensors, key, padding_side="left").to(self.device)
188
+ padded["image_bound"] = [i["image_bound"] for i in input_tensors]
189
+ return padded
190
+
191
+ def _decode(self, inputs_embeds, tokenizer, **kwargs):
192
+ output = self.llm.generate(
193
+ inputs_embeds=inputs_embeds,
194
+ pad_token_id=0,
195
+ eos_token_id=tokenizer.eos_token_id,
196
+ **kwargs
197
+ )
198
+ return self._decode_text(output, tokenizer)
199
+
200
+ def _decode_text(self, result_ids, tokenizer):
201
+ result_text = []
202
+ for result in result_ids:
203
+ result = result[result != 0]
204
+ if result[0] == tokenizer.bos_id:
205
+ result = result[1:]
206
+ if result[-1] == tokenizer.eos_id:
207
+ result = result[:-1]
208
+ result_text.append(tokenizer.decode(result).strip())
209
+ return result_text
210
+
211
+ def get_slice_image_placeholder(self, image, tokenizer):
212
+ image_placeholder = (
213
+ tokenizer.im_start
214
+ + tokenizer.unk_token * self.config.query_num
215
+ + tokenizer.im_end
216
+ )
217
+
218
+ slice_images = []
219
+
220
+ source_image, patches, best_grid = slice_image(
221
+ image,
222
+ self.config.max_slice_nums,
223
+ self.config.scale_resolution,
224
+ self.config.patch_size,
225
+ )
226
+
227
+ slice_images.append(source_image)
228
+ final_placeholder = image_placeholder
229
+
230
+ if len(patches) > 0:
231
+ for i in range(len(patches)):
232
+ for j in range(len(patches[0])):
233
+ slice_images.append(patches[i][j])
234
+
235
+ final_placeholder += get_grid_placeholder(
236
+ tokenizer, best_grid, self.config.query_num
237
+ )
238
+
239
+ return slice_images, final_placeholder
240
+
241
+ def generate(
242
+ self,
243
+ data_list=None,
244
+ img_list=None,
245
+ tokenizer=None,
246
+ max_inp_length: Optional[int] = None,
247
+ vision_hidden_states=None,
248
+ return_vision_hidden_states=False,
249
+ **kwargs
250
+ ):
251
+
252
+ assert data_list is not None
253
+ bs = len(data_list)
254
+ if img_list == None:
255
+ img_list = [[] for i in range(bs)]
256
+ assert bs == len(img_list)
257
+
258
+ model_inputs = self._process_list(tokenizer, data_list, max_inp_length)
259
+
260
+ if vision_hidden_states is None:
261
+ pixel_values = []
262
+ for i in range(bs):
263
+ img_inps = []
264
+ for img in img_list[i]:
265
+ img_inps.append(self.transform(img).to(self.device))
266
+ if img_inps:
267
+ pixel_values.append(img_inps)
268
+ else:
269
+ pixel_values.append([])
270
+ model_inputs["pixel_values"] = pixel_values
271
+ else:
272
+ model_inputs["vision_hidden_states"] = vision_hidden_states
273
+
274
+ with torch.inference_mode():
275
+ (
276
+ model_inputs["inputs_embeds"],
277
+ vision_hidden_states,
278
+ ) = self.get_vllm_embedding(model_inputs)
279
+
280
+ result = self._decode(model_inputs["inputs_embeds"], tokenizer, **kwargs)
281
+
282
+ if return_vision_hidden_states:
283
+ return result, vision_hidden_states
284
+
285
+ return result
286
+
287
+ def chat(
288
+ self,
289
+ image,
290
+ msgs,
291
+ context,
292
+ tokenizer,
293
+ vision_hidden_states=None,
294
+ max_new_tokens=1024,
295
+ sampling=True,
296
+ **kwargs
297
+ ):
298
+ if isinstance(msgs, str):
299
+ msgs = json.loads(msgs)
300
+ # msgs to prompt
301
+ prompt = ""
302
+ for i, msg in enumerate(msgs):
303
+ role = msg["role"]
304
+ content = msg["content"]
305
+ assert role in ["user", "assistant"]
306
+ if i == 0:
307
+ assert role == "user", "The role of first msg should be user"
308
+ if self.config.slice_mode:
309
+ images, final_placeholder = self.get_slice_image_placeholder(
310
+ image, tokenizer
311
+ )
312
+ content = final_placeholder + "\n" + content
313
+ else:
314
+ images = [image]
315
+ content = (
316
+ tokenizer.im_start
317
+ + tokenizer.unk_token * self.config.query_num
318
+ + tokenizer.im_end
319
+ + "\n"
320
+ + content
321
+ )
322
+ prompt += "<用户>" if role == "user" else "<AI>"
323
+ prompt += content
324
+ prompt += "<AI>"
325
+ final_input = prompt
326
+
327
+ if sampling:
328
+ generation_config = {
329
+ "top_p": 0.8,
330
+ "top_k": 100,
331
+ "temperature": 0.7,
332
+ "do_sample": True,
333
+ "repetition_penalty": 1.05
334
+ }
335
+ else:
336
+ generation_config = {
337
+ "num_beams": 3,
338
+ "repetition_penalty": 1.2,
339
+ }
340
+
341
+ generation_config.update(
342
+ (k, kwargs[k]) for k in generation_config.keys() & kwargs.keys()
343
+ )
344
+
345
+ with torch.inference_mode():
346
+ res, vision_hidden_states = self.generate(
347
+ data_list=[final_input],
348
+ max_inp_length=8192,
349
+ img_list=[images],
350
+ tokenizer=tokenizer,
351
+ max_new_tokens=max_new_tokens,
352
+ vision_hidden_states=vision_hidden_states,
353
+ return_vision_hidden_states=True,
354
+ **generation_config
355
+ )
356
+ answer = res[0]
357
+ context = msgs
358
+ context.append({"role": "assistant", "content": answer})
359
+
360
+ return answer, context, generation_config
361
+
362
+
363
+ class LlamaTokenizerWrapper(LlamaTokenizer):
364
+ def __init__(self, **kwargs):
365
+ super().__init__(**kwargs)
366
+ self.im_start = "<image>"
367
+ self.im_end = "</image>"
368
+ self.ref_start = "<ref>"
369
+ self.ref_end = "</ref>"
370
+ self.box_start = "<box>"
371
+ self.box_end = "</box>"
372
+ self.quad_start = "<quad>"
373
+ self.quad_end = "</quad>"
374
+ self.point_start = "<point>"
375
+ self.point_end = "</point>"
376
+ self.slice_start = "<slice>"
377
+ self.slice_end = "</slice>"
378
+
379
+ @property
380
+ def eos_id(self):
381
+ return self.sp_model.eos_id()
382
+
383
+ @property
384
+ def bos_id(self):
385
+ return self.sp_model.bos_id()
386
+
387
+ @property
388
+ def unk_id(self):
389
+ return self.sp_model.unk_id()
390
+
391
+ @property
392
+ def im_start_id(self):
393
+ return self._convert_token_to_id(self.im_start)
394
+
395
+ @property
396
+ def im_end_id(self):
397
+ return self._convert_token_to_id(self.im_end)
398
+
399
+
400
+ def pad(orig_items, key, max_length=None, padding_value=0, padding_side="left"):
401
+ items = []
402
+ if isinstance(orig_items[0][key], list):
403
+ assert isinstance(orig_items[0][key][0], torch.Tensor)
404
+ for it in orig_items:
405
+ for tr in it[key]:
406
+ items.append({key: tr})
407
+ else:
408
+ assert isinstance(orig_items[0][key], torch.Tensor)
409
+ items = orig_items
410
+
411
+ batch_size = len(items)
412
+ shape = items[0][key].shape
413
+ dim = len(shape)
414
+ assert dim <= 3
415
+ if max_length is None:
416
+ max_length = 0
417
+ max_length = max(max_length, max(item[key].shape[-1] for item in items))
418
+ min_length = min(item[key].shape[-1] for item in items)
419
+ dtype = items[0][key].dtype
420
+
421
+ if dim == 1:
422
+ return torch.cat([item[key] for item in items], dim=0)
423
+ elif dim == 2:
424
+ if max_length == min_length:
425
+ return torch.cat([item[key] for item in items], dim=0)
426
+ tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
427
+ else:
428
+ tensor = (
429
+ torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
430
+ + padding_value
431
+ )
432
+
433
+ for i, item in enumerate(items):
434
+ if dim == 2:
435
+ if padding_side == "left":
436
+ tensor[i, -len(item[key][0]) :] = item[key][0].clone()
437
+ else:
438
+ tensor[i, : len(item[key][0])] = item[key][0].clone()
439
+ elif dim == 3:
440
+ if padding_side == "left":
441
+ tensor[i, -len(item[key][0]) :, :] = item[key][0].clone()
442
+ else:
443
+ tensor[i, : len(item[key][0]), :] = item[key][0].clone()
444
+
445
+ return tensor
446
+
447
+
448
+ def slice_image(
449
+ image, max_slice_nums=9, scale_resolution=448, patch_size=14, never_split=False
450
+ ):
451
+ original_size = image.size
452
+ original_width, original_height = original_size
453
+ log_ratio = math.log(original_width / original_height)
454
+ ratio = original_width * original_height / (scale_resolution * scale_resolution)
455
+ multiple = min(math.ceil(ratio), max_slice_nums)
456
+
457
+ source_image = None
458
+ best_grid = None
459
+ patches = []
460
+
461
+ if multiple <= 1 or never_split:
462
+ # dont need to slice, upsample
463
+ best_size = find_best_resize(
464
+ original_size, scale_resolution, patch_size, allow_upscale=True
465
+ )
466
+ source_image = image.resize(best_size, Image.Resampling.BICUBIC)
467
+ else:
468
+ candidate_split_grids_nums = []
469
+ for i in [multiple - 1, multiple, multiple + 1]:
470
+ if i == 1 or i > max_slice_nums:
471
+ continue
472
+ candidate_split_grids_nums.append(i)
473
+
474
+ # source image, down-sampling and ensure divided by patch_size
475
+ best_resize = find_best_resize(original_size, scale_resolution, patch_size)
476
+ source_image = image.copy().resize(best_resize, Image.Resampling.BICUBIC)
477
+ candidate_grids = []
478
+
479
+ # find best grid
480
+ for split_grids_nums in candidate_split_grids_nums:
481
+ m = 1
482
+ while m <= split_grids_nums:
483
+ if split_grids_nums % m == 0:
484
+ candidate_grids.append([m, split_grids_nums // m])
485
+ m += 1
486
+
487
+ best_grid = [1, 1]
488
+ min_error = float("inf")
489
+ for grid in candidate_grids:
490
+ error = abs(log_ratio - math.log(grid[0] / grid[1]))
491
+ if error < min_error:
492
+ best_grid = grid
493
+ min_error = error
494
+
495
+ refine_size = get_refine_size(
496
+ original_size, best_grid, scale_resolution, patch_size, allow_upscale=True
497
+ )
498
+
499
+ refine_image = image.resize(refine_size, Image.Resampling.BICUBIC)
500
+ patches = split_to_patches(refine_image, best_grid)
501
+
502
+ return source_image, patches, best_grid
503
+
504
+
505
+ def ensure_divide(length, patch_size):
506
+ return max(round(length / patch_size) * patch_size, patch_size)
507
+
508
+
509
+ def find_best_resize(original_size, scale_resolution, patch_size, allow_upscale=False):
510
+ width, height = original_size
511
+ if (width * height > scale_resolution * scale_resolution) or allow_upscale:
512
+ r = width / height
513
+ height = int(scale_resolution / math.sqrt(r))
514
+ width = int(height * r)
515
+ best_width = ensure_divide(width, patch_size)
516
+ best_height = ensure_divide(height, patch_size)
517
+ return (best_width, best_height)
518
+
519
+
520
+ def get_refine_size(
521
+ original_size, grid, scale_resolution, patch_size, allow_upscale=False
522
+ ):
523
+ width, height = original_size
524
+ grid_x, grid_y = grid
525
+
526
+ refine_width = ensure_divide(width, grid_x)
527
+ refine_height = ensure_divide(height, grid_y)
528
+
529
+ grid_width = refine_width / grid_x
530
+ grid_height = refine_height / grid_y
531
+
532
+ best_grid_size = find_best_resize(
533
+ (grid_width, grid_height),
534
+ scale_resolution,
535
+ patch_size,
536
+ allow_upscale=allow_upscale,
537
+ )
538
+
539
+ refine_size = (best_grid_size[0] * grid_x, best_grid_size[1] * grid_y)
540
+
541
+ return refine_size
542
+
543
+
544
+ def split_to_patches(image, grid):
545
+ patches = []
546
+ width, height = image.size
547
+ grid_x = int(width / grid[0])
548
+ grid_y = int(height / grid[1])
549
+
550
+ for i in range(0, height, grid_y):
551
+ images = []
552
+ for j in range(0, width, grid_x):
553
+ box = (j, i, j + grid_x, i + grid_y)
554
+ patch = image.crop(box)
555
+ images.append(patch)
556
+ patches.append(images)
557
+
558
+ return patches
559
+
560
+
561
+ def get_grid_placeholder(tokenizer, grid, query_num):
562
+ image_placeholder = (
563
+ tokenizer.im_start + tokenizer.unk_token * query_num + tokenizer.im_end
564
+ )
565
+
566
+ cols = grid[0]
567
+ rows = grid[1]
568
+ slices = []
569
+ for i in range(rows):
570
+ lines = []
571
+ for j in range(cols):
572
+ lines.append(image_placeholder)
573
+ slices.append("".join(lines))
574
+ slice_placeholder = tokenizer.slice_start + "\n".join(slices) + tokenizer.slice_end
575
+ return slice_placeholder
resampler.py ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from collections import OrderedDict
7
+ import math
8
+ import requests
9
+ from io import BytesIO
10
+ from functools import partial
11
+ from PIL import Image
12
+ from typing import Callable, Optional, Sequence, Tuple, List, Union
13
+ import numpy as np
14
+
15
+ import torch
16
+ from torch import nn
17
+ from torch.nn import functional as F
18
+ from torch.nn.init import trunc_normal_
19
+ from torchvision import transforms
20
+ from torchvision.transforms import InterpolationMode
21
+
22
+ def get_abs_pos(abs_pos, tgt_size):
23
+ # abs_pos: L, C
24
+ # tgt_size: (H, W)
25
+ # return: M, C
26
+ src_size = int(math.sqrt(abs_pos.size(0)))
27
+ # tgt_size = int(math.sqrt(tgt_size))
28
+ dtype = abs_pos.dtype
29
+
30
+ return F.interpolate(
31
+ abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
32
+ size=(tgt_size[0], tgt_size[1]),
33
+ mode="bicubic",
34
+ align_corners=False,
35
+ ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
36
+
37
+
38
+ # https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
39
+ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
40
+ """
41
+ grid_size: int of the grid height and width
42
+ return:
43
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
44
+ """
45
+ if isinstance(grid_size, int):
46
+ grid_h_size, grid_w_size = grid_size, grid_size
47
+ else:
48
+ grid_h_size, grid_w_size = grid_size[0], grid_size[1]
49
+
50
+ grid_h = np.arange(grid_h_size, dtype=np.float32)
51
+ grid_w = np.arange(grid_w_size, dtype=np.float32)
52
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
53
+ grid = np.stack(grid, axis=0)
54
+
55
+ grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
56
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
57
+ if cls_token:
58
+ pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
59
+ return pos_embed
60
+
61
+
62
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
63
+ assert embed_dim % 2 == 0
64
+
65
+ # use half of dimensions to encode grid_h
66
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
67
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
68
+
69
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
70
+ return emb
71
+
72
+
73
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
74
+ """
75
+ embed_dim: output dimension for each position
76
+ pos: a list of positions to be encoded: size (M,)
77
+ out: (M, D)
78
+ """
79
+ assert embed_dim % 2 == 0
80
+ omega = np.arange(embed_dim // 2, dtype=np.float32)
81
+ omega /= embed_dim / 2.
82
+ omega = 1. / 10000 ** omega # (D/2,)
83
+
84
+ pos = pos.reshape(-1) # (M,)
85
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
86
+
87
+ emb_sin = np.sin(out) # (M, D/2)
88
+ emb_cos = np.cos(out) # (M, D/2)
89
+
90
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
91
+ return emb
92
+
93
+
94
+ class Resampler(nn.Module):
95
+ """
96
+ A 2D perceiver-resampler network with one cross attention layers by
97
+ (grid_size**2) learnable queries and 2d sincos pos_emb
98
+ Outputs:
99
+ A tensor with the shape of (grid_size**2, embed_dim)
100
+ """
101
+
102
+ def __init__(
103
+ self,
104
+ grid_size,
105
+ embed_dim,
106
+ num_heads,
107
+ kv_dim=None,
108
+ norm_layer=partial(nn.LayerNorm, eps=1e-6),
109
+ adaptive=False
110
+ ):
111
+ super().__init__()
112
+ self.num_queries = grid_size ** 2
113
+ self.embed_dim = embed_dim
114
+ self.num_heads = num_heads
115
+ self.adaptive = adaptive
116
+
117
+ self.pos_embed = nn.Parameter(
118
+ torch.from_numpy(get_2d_sincos_pos_embed(embed_dim, grid_size)).float()
119
+ ).requires_grad_(False)
120
+
121
+ self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
122
+ trunc_normal_(self.query, std=.02)
123
+
124
+ if kv_dim is not None and kv_dim != embed_dim:
125
+ self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
126
+ else:
127
+ self.kv_proj = nn.Identity()
128
+
129
+ self.attn = nn.MultiheadAttention(embed_dim, num_heads)
130
+ self.ln_q = norm_layer(embed_dim)
131
+ self.ln_kv = norm_layer(embed_dim)
132
+
133
+ self.ln_post = norm_layer(embed_dim)
134
+ self.proj = nn.Parameter((embed_dim ** -0.5) * torch.randn(embed_dim, embed_dim))
135
+
136
+ self.apply(self._init_weights)
137
+
138
+ def _init_weights(self, m):
139
+ if isinstance(m, nn.Linear):
140
+ trunc_normal_(m.weight, std=.02)
141
+ if isinstance(m, nn.Linear) and m.bias is not None:
142
+ nn.init.constant_(m.bias, 0)
143
+ elif isinstance(m, nn.LayerNorm):
144
+ nn.init.constant_(m.bias, 0)
145
+ nn.init.constant_(m.weight, 1.0)
146
+
147
+ def forward(self, x, tgt_size=None, attn_mask=None):
148
+ if self.adaptive:
149
+ pos_embed = torch.Tensor(get_2d_sincos_pos_embed(self.embed_dim, tgt_size)).float().to(device=x.device, dtype=x.dtype)
150
+ else:
151
+ pos_embed = get_abs_pos(self.pos_embed, tgt_size)
152
+
153
+ x = self.kv_proj(x)
154
+ x = self.ln_kv(x).permute(1, 0, 2)
155
+
156
+ N = x.shape[1]
157
+ q = self.ln_q(self.query)
158
+ out = self.attn(
159
+ self._repeat(q, N) + self.pos_embed.unsqueeze(1),
160
+ x + pos_embed.unsqueeze(1),
161
+ x,
162
+ attn_mask=attn_mask)[0]
163
+ x = out.permute(1, 0, 2)
164
+
165
+ x = self.ln_post(x)
166
+ x = x @ self.proj
167
+ return x
168
+
169
+ def _repeat(self, query, N: int):
170
+ return query.unsqueeze(1).repeat(1, N, 1)
special_tokens_map.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<image>",
4
+ "</image>",
5
+ "<ref>",
6
+ "</ref>",
7
+ "<box>",
8
+ "</box>",
9
+ "<quad>",
10
+ "</quad>",
11
+ "<point>",
12
+ "</point>",
13
+ "<slice>",
14
+ "</slice>"
15
+ ],
16
+ "bos_token": {
17
+ "content": "<s>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "eos_token": {
24
+ "content": "</s>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "pad_token": "<unk>",
31
+ "unk_token": {
32
+ "content": "<unk>",
33
+ "lstrip": false,
34
+ "normalized": false,
35
+ "rstrip": false,
36
+ "single_word": false
37
+ }
38
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "101": {
30
+ "content": "<image>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "102": {
38
+ "content": "</image>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "103": {
46
+ "content": "<ref>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "104": {
54
+ "content": "</ref>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "105": {
62
+ "content": "<box>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "106": {
70
+ "content": "</box>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "107": {
78
+ "content": "<quad>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "108": {
86
+ "content": "</quad>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "109": {
94
+ "content": "<point>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "110": {
102
+ "content": "</point>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "111": {
110
+ "content": "<slice>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "112": {
118
+ "content": "</slice>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": true
124
+ }
125
+ },
126
+ "additional_special_tokens": [
127
+ "<image>",
128
+ "</image>",
129
+ "<ref>",
130
+ "</ref>",
131
+ "<box>",
132
+ "</box>",
133
+ "<quad>",
134
+ "</quad>",
135
+ "<point>",
136
+ "</point>",
137
+ "<slice>",
138
+ "</slice>"
139
+ ],
140
+ "auto_map": {
141
+ "AutoTokenizer": [
142
+ "modeling_minicpmv.LlamaTokenizerWrapper",
143
+ null
144
+ ]
145
+ },
146
+ "bos_token": "<s>",
147
+ "clean_up_tokenization_spaces": false,
148
+ "eos_token": "</s>",
149
+ "legacy": true,
150
+ "model_max_length": 2048,
151
+ "pad_token": "<unk>",
152
+ "padding_side": "right",
153
+ "sp_model_kwargs": {},
154
+ "spaces_between_special_tokens": false,
155
+ "tokenizer_class": "LlamaTokenizerWrapper",
156
+ "truncation_side": "right",
157
+ "unk_token": "<unk>",
158
+ "use_default_system_prompt": false
159
+ }