zechunliu commited on
Commit
5c139f5
1 Parent(s): 7edef26

Upload 6 files

Browse files
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MobileLLMForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_mobilellm.MobileLLMConfig",
9
+ "AutoModelForCausalLM": "modeling_mobilellm.MobileLLMForCausalLM"
10
+ },
11
+ "bos_token_id": 1,
12
+ "eos_token_id": 2,
13
+ "head_dim": 64,
14
+ "hidden_act": "silu",
15
+ "hidden_size": 960,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 2560,
18
+ "layer_sharing": false,
19
+ "max_position_embeddings": 2048,
20
+ "mlp_bias": false,
21
+ "model_type": "mobilellm",
22
+ "num_attention_heads": 15,
23
+ "num_hidden_layers": 32,
24
+ "num_key_value_heads": 5,
25
+ "pretraining_tp": 1,
26
+ "rms_norm_eps": 1e-05,
27
+ "rope_scaling": null,
28
+ "rope_theta": 10000.0,
29
+ "share_embedding": true,
30
+ "tie_word_embeddings": false,
31
+ "torch_dtype": "float16",
32
+ "transformers_version": "4.41.2",
33
+ "use_cache": true,
34
+ "vocab_size": 32000
35
+ }
configuration_mobilellm.py ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ from transformers.configuration_utils import PretrainedConfig
8
+
9
+
10
+ class MobileLLMConfig(PretrainedConfig):
11
+
12
+ model_type = "mobilellm"
13
+ keys_to_ignore_at_inference = ["past_key_values"]
14
+
15
+ def __init__(
16
+ self,
17
+ vocab_size=32000,
18
+ hidden_size=4096,
19
+ intermediate_size=11008,
20
+ num_hidden_layers=32,
21
+ num_attention_heads=32,
22
+ num_key_value_heads=None,
23
+ hidden_act="silu",
24
+ max_position_embeddings=2048,
25
+ initializer_range=0.02,
26
+ rms_norm_eps=1e-6,
27
+ use_cache=True,
28
+ pad_token_id=None,
29
+ bos_token_id=1,
30
+ eos_token_id=2,
31
+ pretraining_tp=1,
32
+ tie_word_embeddings=False,
33
+ rope_theta=10000.0,
34
+ rope_scaling=None,
35
+ attention_bias=False,
36
+ attention_dropout=0.0,
37
+ mlp_bias=False,
38
+ head_dim=None,
39
+ share_embedding=True,
40
+ layer_sharing=False,
41
+ **kwargs,
42
+ ):
43
+ self.vocab_size = vocab_size
44
+ self.max_position_embeddings = max_position_embeddings
45
+ self.hidden_size = hidden_size
46
+ self.intermediate_size = intermediate_size
47
+ self.num_hidden_layers = num_hidden_layers
48
+ self.num_attention_heads = num_attention_heads
49
+
50
+ # for backward compatibility
51
+ if num_key_value_heads is None:
52
+ num_key_value_heads = num_attention_heads
53
+
54
+ self.num_key_value_heads = num_key_value_heads
55
+ self.hidden_act = hidden_act
56
+ self.initializer_range = initializer_range
57
+ self.rms_norm_eps = rms_norm_eps
58
+ self.pretraining_tp = pretraining_tp
59
+ self.use_cache = use_cache
60
+ self.rope_theta = rope_theta
61
+ self.rope_scaling = rope_scaling
62
+ self.attention_bias = attention_bias
63
+ self.attention_dropout = attention_dropout
64
+ self.mlp_bias = mlp_bias
65
+ self.head_dim = head_dim if head_dim is not None else self.hidden_size // self.num_attention_heads
66
+ self.share_embedding = share_embedding
67
+ self.layer_sharing = layer_sharing
68
+ # Validate the correctness of rotary position embeddings parameters
69
+ # BC: if there is a 'type' field, copy it it to 'rope_type'.
70
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
71
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
72
+
73
+ super().__init__(
74
+ pad_token_id=pad_token_id,
75
+ bos_token_id=bos_token_id,
76
+ eos_token_id=eos_token_id,
77
+ tie_word_embeddings=tie_word_embeddings,
78
+ **kwargs,
79
+ )
modeling_mobilellm.py ADDED
@@ -0,0 +1,1651 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ # All rights reserved.
4
+ #
5
+ # This source code is licensed under the license found in the
6
+ # LICENSE file in the root directory of this source tree.
7
+
8
+ # coding=utf-8
9
+ # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
10
+ #
11
+ # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
12
+ # and OPT implementations in this library. It has been modified from its
13
+ # original forms to accommodate minor architectural differences compared
14
+ # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
15
+ #
16
+ # Licensed under the Apache License, Version 2.0 (the "License");
17
+ # you may not use this file except in compliance with the License.
18
+ # You may obtain a copy of the License at
19
+ #
20
+ # http://www.apache.org/licenses/LICENSE-2.0
21
+ #
22
+ # Unless required by applicable law or agreed to in writing, software
23
+ # distributed under the License is distributed on an "AS IS" BASIS,
24
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25
+ # See the License for the specific language governing permissions and
26
+ # limitations under the License.
27
+ import math
28
+ from typing import List, Optional, Tuple, Union
29
+
30
+ import torch
31
+ import torch.nn.functional as F
32
+ import torch.utils.checkpoint
33
+ from torch import nn
34
+ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
35
+
36
+ from transformers.activations import ACT2FN
37
+ from transformers.cache_utils import Cache, DynamicCache, StaticCache
38
+ from transformers.modeling_attn_mask_utils import AttentionMaskConverter
39
+ from transformers.modeling_outputs import (
40
+ BaseModelOutputWithPast,
41
+ CausalLMOutputWithPast,
42
+ QuestionAnsweringModelOutput,
43
+ SequenceClassifierOutputWithPast,
44
+ TokenClassifierOutput,
45
+ )
46
+ from transformers.modeling_utils import PreTrainedModel
47
+ from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
48
+ from transformers.utils import (
49
+ add_start_docstrings,
50
+ add_start_docstrings_to_model_forward,
51
+ is_flash_attn_2_available,
52
+ is_flash_attn_greater_or_equal_2_10,
53
+ logging,
54
+ replace_return_docstrings,
55
+ )
56
+ from .configuration_mobilellm import MobileLLMConfig
57
+
58
+
59
+ if is_flash_attn_2_available():
60
+ from flash_attn import flash_attn_func, flash_attn_varlen_func
61
+ from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa
62
+
63
+
64
+ logger = logging.get_logger(__name__)
65
+
66
+ _CONFIG_FOR_DOC = "MobileLLMConfig"
67
+
68
+
69
+ def _get_unpad_data(attention_mask):
70
+ seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
71
+ indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
72
+ max_seqlen_in_batch = seqlens_in_batch.max().item()
73
+ cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
74
+ return (
75
+ indices,
76
+ cu_seqlens,
77
+ max_seqlen_in_batch,
78
+ )
79
+
80
+
81
+ class LlamaRMSNorm(nn.Module):
82
+ def __init__(self, hidden_size, eps=1e-6):
83
+ """
84
+ LlamaRMSNorm is equivalent to T5LayerNorm
85
+ """
86
+ super().__init__()
87
+ self.weight = nn.Parameter(torch.ones(hidden_size))
88
+ self.variance_epsilon = eps
89
+
90
+ def forward(self, hidden_states):
91
+ input_dtype = hidden_states.dtype
92
+ hidden_states = hidden_states.to(torch.float32)
93
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
94
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
95
+ return self.weight * hidden_states.to(input_dtype)
96
+
97
+
98
+ ALL_LAYERNORM_LAYERS.append(LlamaRMSNorm)
99
+
100
+
101
+ class LlamaRotaryEmbedding(nn.Module):
102
+ def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
103
+ super().__init__()
104
+ self.scaling_factor = scaling_factor
105
+ self.dim = dim
106
+ self.max_position_embeddings = max_position_embeddings
107
+ self.base = base
108
+ inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim))
109
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
110
+ # For BC we register cos and sin cached
111
+ self.max_seq_len_cached = max_position_embeddings
112
+
113
+ @torch.no_grad()
114
+ def forward(self, x, position_ids):
115
+ # x: [bs, num_attention_heads, seq_len, head_size]
116
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
117
+ position_ids_expanded = position_ids[:, None, :].float()
118
+ # Force float32 since bfloat16 loses precision on long contexts
119
+ # See https://github.com/huggingface/transformers/pull/29285
120
+ device_type = x.device.type
121
+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
122
+ with torch.autocast(device_type=device_type, enabled=False):
123
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
124
+ emb = torch.cat((freqs, freqs), dim=-1)
125
+ cos = emb.cos()
126
+ sin = emb.sin()
127
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
128
+
129
+
130
+ class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):
131
+ """LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
132
+
133
+ def forward(self, x, position_ids):
134
+ # difference to the original RoPE: a scaling factor is aplied to the position ids
135
+ position_ids = position_ids.float() / self.scaling_factor
136
+ cos, sin = super().forward(x, position_ids)
137
+ return cos, sin
138
+
139
+
140
+ class LlamaDynamicNTKScalingRotaryEmbedding(LlamaRotaryEmbedding):
141
+ """LlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
142
+
143
+ def forward(self, x, position_ids):
144
+ # difference to the original RoPE: inv_freq is recomputed when the sequence length > original length
145
+ seq_len = torch.max(position_ids) + 1
146
+ if seq_len > self.max_position_embeddings:
147
+ base = self.base * (
148
+ (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)
149
+ ) ** (self.dim / (self.dim - 2))
150
+ inv_freq = 1.0 / (
151
+ base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(x.device) / self.dim)
152
+ )
153
+ self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: this may break with compilation
154
+
155
+ cos, sin = super().forward(x, position_ids)
156
+ return cos, sin
157
+
158
+
159
+ def rotate_half(x):
160
+ """Rotates half the hidden dims of the input."""
161
+ x1 = x[..., : x.shape[-1] // 2]
162
+ x2 = x[..., x.shape[-1] // 2 :]
163
+ return torch.cat((-x2, x1), dim=-1)
164
+
165
+
166
+ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
167
+ """Applies Rotary Position Embedding to the query and key tensors.
168
+
169
+ Args:
170
+ q (`torch.Tensor`): The query tensor.
171
+ k (`torch.Tensor`): The key tensor.
172
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
173
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
174
+ position_ids (`torch.Tensor`, *optional*):
175
+ Deprecated and unused.
176
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
177
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
178
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
179
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
180
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
181
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
182
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
183
+ Returns:
184
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
185
+ """
186
+ cos = cos.unsqueeze(unsqueeze_dim)
187
+ sin = sin.unsqueeze(unsqueeze_dim)
188
+ q_embed = (q * cos) + (rotate_half(q) * sin)
189
+ k_embed = (k * cos) + (rotate_half(k) * sin)
190
+ return q_embed, k_embed
191
+
192
+
193
+ class LlamaMLP(nn.Module):
194
+ def __init__(self, config):
195
+ super().__init__()
196
+ self.config = config
197
+ self.hidden_size = config.hidden_size
198
+ self.intermediate_size = config.intermediate_size
199
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
200
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
201
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias)
202
+ self.act_fn = ACT2FN[config.hidden_act]
203
+
204
+ def forward(self, x):
205
+ if self.config.pretraining_tp > 1:
206
+ slice = self.intermediate_size // self.config.pretraining_tp
207
+ gate_proj_slices = self.gate_proj.weight.split(slice, dim=0)
208
+ up_proj_slices = self.up_proj.weight.split(slice, dim=0)
209
+ down_proj_slices = self.down_proj.weight.split(slice, dim=1)
210
+
211
+ gate_proj = torch.cat(
212
+ [F.linear(x, gate_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1
213
+ )
214
+ up_proj = torch.cat([F.linear(x, up_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1)
215
+
216
+ intermediate_states = (self.act_fn(gate_proj) * up_proj).split(slice, dim=2)
217
+ down_proj = [
218
+ F.linear(intermediate_states[i], down_proj_slices[i]) for i in range(self.config.pretraining_tp)
219
+ ]
220
+ down_proj = sum(down_proj)
221
+ else:
222
+ down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
223
+
224
+ return down_proj
225
+
226
+
227
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
228
+ """
229
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
230
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
231
+ """
232
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
233
+ if n_rep == 1:
234
+ return hidden_states
235
+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
236
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
237
+
238
+
239
+ class LlamaAttention(nn.Module):
240
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
241
+
242
+ def __init__(self, config: MobileLLMConfig, layer_idx: Optional[int] = None):
243
+ super().__init__()
244
+ self.config = config
245
+ self.layer_idx = layer_idx
246
+ if layer_idx is None:
247
+ logger.warning_once(
248
+ f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will "
249
+ "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` "
250
+ "when creating this class."
251
+ )
252
+
253
+ self.attention_dropout = config.attention_dropout
254
+ self.hidden_size = config.hidden_size
255
+ self.num_heads = config.num_attention_heads
256
+ self.head_dim = self.hidden_size // self.num_heads
257
+ self.num_key_value_heads = config.num_key_value_heads
258
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
259
+ self.max_position_embeddings = config.max_position_embeddings
260
+ self.rope_theta = config.rope_theta
261
+ self.is_causal = True
262
+
263
+ if (self.head_dim * self.num_heads) != self.hidden_size:
264
+ raise ValueError(
265
+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
266
+ f" and `num_heads`: {self.num_heads})."
267
+ )
268
+
269
+ self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias)
270
+ self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
271
+ self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
272
+ self.o_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=config.attention_bias)
273
+ self._init_rope()
274
+
275
+ def _init_rope(self):
276
+ if self.config.rope_scaling is None:
277
+ self.rotary_emb = LlamaRotaryEmbedding(
278
+ self.head_dim,
279
+ max_position_embeddings=self.max_position_embeddings,
280
+ base=self.rope_theta,
281
+ )
282
+ else:
283
+ scaling_type = self.config.rope_scaling["type"]
284
+ scaling_factor = self.config.rope_scaling["factor"]
285
+ if scaling_type == "linear":
286
+ self.rotary_emb = LlamaLinearScalingRotaryEmbedding(
287
+ self.head_dim,
288
+ max_position_embeddings=self.max_position_embeddings,
289
+ scaling_factor=scaling_factor,
290
+ base=self.rope_theta,
291
+ )
292
+ elif scaling_type == "dynamic":
293
+ self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding(
294
+ self.head_dim,
295
+ max_position_embeddings=self.max_position_embeddings,
296
+ scaling_factor=scaling_factor,
297
+ base=self.rope_theta,
298
+ )
299
+ else:
300
+ raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
301
+
302
+ def forward(
303
+ self,
304
+ hidden_states: torch.Tensor,
305
+ attention_mask: Optional[torch.Tensor] = None,
306
+ position_ids: Optional[torch.LongTensor] = None,
307
+ past_key_value: Optional[Cache] = None,
308
+ output_attentions: bool = False,
309
+ use_cache: bool = False,
310
+ cache_position: Optional[torch.LongTensor] = None,
311
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
312
+ bsz, q_len, _ = hidden_states.size()
313
+
314
+ if self.config.pretraining_tp > 1:
315
+ key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp
316
+ query_slices = self.q_proj.weight.split(
317
+ (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0
318
+ )
319
+ key_slices = self.k_proj.weight.split(key_value_slicing, dim=0)
320
+ value_slices = self.v_proj.weight.split(key_value_slicing, dim=0)
321
+
322
+ query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)]
323
+ query_states = torch.cat(query_states, dim=-1)
324
+
325
+ key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)]
326
+ key_states = torch.cat(key_states, dim=-1)
327
+
328
+ value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)]
329
+ value_states = torch.cat(value_states, dim=-1)
330
+
331
+ else:
332
+ query_states = self.q_proj(hidden_states)
333
+ key_states = self.k_proj(hidden_states)
334
+ value_states = self.v_proj(hidden_states)
335
+
336
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
337
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
338
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
339
+
340
+ cos, sin = self.rotary_emb(value_states, position_ids)
341
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
342
+
343
+ if past_key_value is not None:
344
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
345
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
346
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
347
+
348
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
349
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
350
+
351
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
352
+
353
+ if attention_mask is not None: # no matter the length, we just slice it
354
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
355
+ attn_weights = attn_weights + causal_mask
356
+
357
+ # upcast attention to fp32
358
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
359
+ attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
360
+ attn_output = torch.matmul(attn_weights, value_states)
361
+
362
+ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
363
+ raise ValueError(
364
+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
365
+ f" {attn_output.size()}"
366
+ )
367
+
368
+ attn_output = attn_output.transpose(1, 2).contiguous()
369
+
370
+ attn_output = attn_output.reshape(bsz, q_len, -1)
371
+
372
+ if self.config.pretraining_tp > 1:
373
+ attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, dim=2)
374
+ o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, dim=1)
375
+ attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp)])
376
+ else:
377
+ attn_output = self.o_proj(attn_output)
378
+
379
+ if not output_attentions:
380
+ attn_weights = None
381
+
382
+ return attn_output, attn_weights, past_key_value
383
+
384
+
385
+ class LlamaFlashAttention2(LlamaAttention):
386
+ """
387
+ Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays
388
+ untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
389
+ flash attention and deal with padding tokens in case the input contains any of them.
390
+ """
391
+
392
+ def __init__(self, *args, **kwargs):
393
+ super().__init__(*args, **kwargs)
394
+
395
+ # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
396
+ # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
397
+ # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
398
+ self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
399
+
400
+ def forward(
401
+ self,
402
+ hidden_states: torch.Tensor,
403
+ attention_mask: Optional[torch.LongTensor] = None,
404
+ position_ids: Optional[torch.LongTensor] = None,
405
+ past_key_value: Optional[Cache] = None,
406
+ output_attentions: bool = False,
407
+ use_cache: bool = False,
408
+ cache_position: Optional[torch.LongTensor] = None,
409
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
410
+ if isinstance(past_key_value, StaticCache):
411
+ raise ValueError(
412
+ "`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` "
413
+ "make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformers"
414
+ )
415
+
416
+ output_attentions = False
417
+
418
+ bsz, q_len, _ = hidden_states.size()
419
+
420
+ query_states = self.q_proj(hidden_states)
421
+ key_states = self.k_proj(hidden_states)
422
+ value_states = self.v_proj(hidden_states)
423
+
424
+ # Flash attention requires the input to have the shape
425
+ # batch_size x seq_length x head_dim x hidden_dim
426
+ # therefore we just need to keep the original shape
427
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
428
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
429
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
430
+
431
+ cos, sin = self.rotary_emb(value_states, position_ids)
432
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
433
+
434
+ if past_key_value is not None:
435
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
436
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
437
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
438
+
439
+ # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
440
+ # to be able to avoid many of these transpose/reshape/view.
441
+ query_states = query_states.transpose(1, 2)
442
+ key_states = key_states.transpose(1, 2)
443
+ value_states = value_states.transpose(1, 2)
444
+
445
+ dropout_rate = self.attention_dropout if self.training else 0.0
446
+
447
+ # In PEFT, usually we cast the layer norms in float32 for training stability reasons
448
+ # therefore the input hidden states gets silently casted in float32. Hence, we need
449
+ # cast them back in the correct dtype just to be sure everything works as expected.
450
+ # This might slowdown training & inference so it is recommended to not cast the LayerNorms
451
+ # in fp32. (LlamaRMSNorm handles it correctly)
452
+
453
+ input_dtype = query_states.dtype
454
+ if input_dtype == torch.float32:
455
+ if torch.is_autocast_enabled():
456
+ target_dtype = torch.get_autocast_gpu_dtype()
457
+ # Handle the case where the model is quantized
458
+ elif hasattr(self.config, "_pre_quantization_dtype"):
459
+ target_dtype = self.config._pre_quantization_dtype
460
+ else:
461
+ target_dtype = self.q_proj.weight.dtype
462
+
463
+ logger.warning_once(
464
+ f"The input hidden states seems to be silently casted in float32, this might be related to"
465
+ f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
466
+ f" {target_dtype}."
467
+ )
468
+
469
+ query_states = query_states.to(target_dtype)
470
+ key_states = key_states.to(target_dtype)
471
+ value_states = value_states.to(target_dtype)
472
+
473
+ attn_output = self._flash_attention_forward(
474
+ query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
475
+ )
476
+
477
+ attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
478
+ attn_output = self.o_proj(attn_output)
479
+
480
+ if not output_attentions:
481
+ attn_weights = None
482
+
483
+ return attn_output, attn_weights, past_key_value
484
+
485
+ def _flash_attention_forward(
486
+ self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
487
+ ):
488
+ """
489
+ Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
490
+ first unpad the input, then computes the attention scores and pad the final attention scores.
491
+
492
+ Args:
493
+ query_states (`torch.Tensor`):
494
+ Input query states to be passed to Flash Attention API
495
+ key_states (`torch.Tensor`):
496
+ Input key states to be passed to Flash Attention API
497
+ value_states (`torch.Tensor`):
498
+ Input value states to be passed to Flash Attention API
499
+ attention_mask (`torch.Tensor`):
500
+ The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
501
+ position of padding tokens and 1 for the position of non-padding tokens.
502
+ dropout (`float`):
503
+ Attention dropout
504
+ softmax_scale (`float`, *optional*):
505
+ The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
506
+ """
507
+ if not self._flash_attn_uses_top_left_mask:
508
+ causal = self.is_causal
509
+ else:
510
+ # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
511
+ causal = self.is_causal and query_length != 1
512
+
513
+ # Contains at least one padding token in the sequence
514
+ if attention_mask is not None:
515
+ batch_size = query_states.shape[0]
516
+ query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
517
+ query_states, key_states, value_states, attention_mask, query_length
518
+ )
519
+
520
+ cu_seqlens_q, cu_seqlens_k = cu_seq_lens
521
+ max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
522
+
523
+ attn_output_unpad = flash_attn_varlen_func(
524
+ query_states,
525
+ key_states,
526
+ value_states,
527
+ cu_seqlens_q=cu_seqlens_q,
528
+ cu_seqlens_k=cu_seqlens_k,
529
+ max_seqlen_q=max_seqlen_in_batch_q,
530
+ max_seqlen_k=max_seqlen_in_batch_k,
531
+ dropout_p=dropout,
532
+ softmax_scale=softmax_scale,
533
+ causal=causal,
534
+ )
535
+
536
+ attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
537
+ else:
538
+ attn_output = flash_attn_func(
539
+ query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
540
+ )
541
+
542
+ return attn_output
543
+
544
+ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
545
+ indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
546
+ batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
547
+
548
+ key_layer = index_first_axis(
549
+ key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
550
+ )
551
+ value_layer = index_first_axis(
552
+ value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
553
+ )
554
+ if query_length == kv_seq_len:
555
+ query_layer = index_first_axis(
556
+ query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
557
+ )
558
+ cu_seqlens_q = cu_seqlens_k
559
+ max_seqlen_in_batch_q = max_seqlen_in_batch_k
560
+ indices_q = indices_k
561
+ elif query_length == 1:
562
+ max_seqlen_in_batch_q = 1
563
+ cu_seqlens_q = torch.arange(
564
+ batch_size + 1, dtype=torch.int32, device=query_layer.device
565
+ ) # There is a memcpy here, that is very bad.
566
+ indices_q = cu_seqlens_q[:-1]
567
+ query_layer = query_layer.squeeze(1)
568
+ else:
569
+ # The -q_len: slice assumes left padding.
570
+ attention_mask = attention_mask[:, -query_length:]
571
+ query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
572
+
573
+ return (
574
+ query_layer,
575
+ key_layer,
576
+ value_layer,
577
+ indices_q,
578
+ (cu_seqlens_q, cu_seqlens_k),
579
+ (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
580
+ )
581
+
582
+
583
+ class LlamaSdpaAttention(LlamaAttention):
584
+ """
585
+ Llama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
586
+ `LlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
587
+ SDPA API.
588
+ """
589
+
590
+ # Adapted from LlamaAttention.forward
591
+ def forward(
592
+ self,
593
+ hidden_states: torch.Tensor,
594
+ attention_mask: Optional[torch.Tensor] = None,
595
+ position_ids: Optional[torch.LongTensor] = None,
596
+ past_key_value: Optional[Cache] = None,
597
+ output_attentions: bool = False,
598
+ use_cache: bool = False,
599
+ cache_position: Optional[torch.LongTensor] = None,
600
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
601
+ if output_attentions:
602
+ # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
603
+ logger.warning_once(
604
+ "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
605
+ 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
606
+ )
607
+ return super().forward(
608
+ hidden_states=hidden_states,
609
+ attention_mask=attention_mask,
610
+ position_ids=position_ids,
611
+ past_key_value=past_key_value,
612
+ output_attentions=output_attentions,
613
+ use_cache=use_cache,
614
+ cache_position=cache_position,
615
+ )
616
+
617
+ bsz, q_len, _ = hidden_states.size()
618
+
619
+ query_states = self.q_proj(hidden_states)
620
+ key_states = self.k_proj(hidden_states)
621
+ value_states = self.v_proj(hidden_states)
622
+
623
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
624
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
625
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
626
+
627
+ cos, sin = self.rotary_emb(value_states, position_ids)
628
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
629
+
630
+ if past_key_value is not None:
631
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
632
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
633
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
634
+
635
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
636
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
637
+
638
+ causal_mask = attention_mask
639
+ if attention_mask is not None:
640
+ causal_mask = causal_mask[:, :, :, : key_states.shape[-2]]
641
+
642
+ # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
643
+ # Reference: https://github.com/pytorch/pytorch/issues/112577.
644
+ if query_states.device.type == "cuda" and causal_mask is not None:
645
+ query_states = query_states.contiguous()
646
+ key_states = key_states.contiguous()
647
+ value_states = value_states.contiguous()
648
+
649
+ # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment
650
+ # in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling.
651
+ is_causal = True if causal_mask is None and q_len > 1 else False
652
+
653
+ attn_output = torch.nn.functional.scaled_dot_product_attention(
654
+ query_states,
655
+ key_states,
656
+ value_states,
657
+ attn_mask=causal_mask,
658
+ dropout_p=self.attention_dropout if self.training else 0.0,
659
+ is_causal=is_causal,
660
+ )
661
+
662
+ attn_output = attn_output.transpose(1, 2).contiguous()
663
+ attn_output = attn_output.view(bsz, q_len, -1)
664
+
665
+ attn_output = self.o_proj(attn_output)
666
+
667
+ return attn_output, None, past_key_value
668
+
669
+
670
+ LLAMA_ATTENTION_CLASSES = {
671
+ "eager": LlamaAttention,
672
+ "flash_attention_2": LlamaFlashAttention2,
673
+ "sdpa": LlamaSdpaAttention,
674
+ }
675
+
676
+
677
+ class LlamaDecoderLayer(nn.Module):
678
+ def __init__(self, config: MobileLLMConfig, layer_idx: int):
679
+ super().__init__()
680
+ self.hidden_size = config.hidden_size
681
+
682
+ self.self_attn = LLAMA_ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
683
+
684
+ self.mlp = LlamaMLP(config)
685
+ self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
686
+ self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
687
+
688
+ def forward(
689
+ self,
690
+ hidden_states: torch.Tensor,
691
+ attention_mask: Optional[torch.Tensor] = None,
692
+ position_ids: Optional[torch.LongTensor] = None,
693
+ past_key_value: Optional[Cache] = None,
694
+ output_attentions: Optional[bool] = False,
695
+ use_cache: Optional[bool] = False,
696
+ cache_position: Optional[torch.LongTensor] = None,
697
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
698
+ """
699
+ Args:
700
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
701
+ attention_mask (`torch.FloatTensor`, *optional*):
702
+ attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
703
+ query_sequence_length, key_sequence_length)` if default attention is used.
704
+ output_attentions (`bool`, *optional*):
705
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
706
+ returned tensors for more detail.
707
+ use_cache (`bool`, *optional*):
708
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
709
+ (see `past_key_values`).
710
+ past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
711
+ """
712
+ residual = hidden_states
713
+
714
+ hidden_states = self.input_layernorm(hidden_states)
715
+
716
+ # Self Attention
717
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
718
+ hidden_states=hidden_states,
719
+ attention_mask=attention_mask,
720
+ position_ids=position_ids,
721
+ past_key_value=past_key_value,
722
+ output_attentions=output_attentions,
723
+ use_cache=use_cache,
724
+ cache_position=cache_position,
725
+ )
726
+ hidden_states = residual + hidden_states
727
+
728
+ # Fully Connected
729
+ residual = hidden_states
730
+ hidden_states = self.post_attention_layernorm(hidden_states)
731
+ hidden_states = self.mlp(hidden_states)
732
+ hidden_states = residual + hidden_states
733
+
734
+ outputs = (hidden_states,)
735
+
736
+ if output_attentions:
737
+ outputs += (self_attn_weights,)
738
+
739
+ if use_cache:
740
+ outputs += (present_key_value,)
741
+
742
+ return outputs
743
+
744
+
745
+ LLAMA_START_DOCSTRING = r"""
746
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
747
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
748
+ etc.)
749
+
750
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
751
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
752
+ and behavior.
753
+
754
+ Parameters:
755
+ config ([`LlamaConfig`]):
756
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
757
+ load the weights associated with the model, only the configuration. Check out the
758
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
759
+ """
760
+
761
+
762
+ @add_start_docstrings(
763
+ "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
764
+ LLAMA_START_DOCSTRING,
765
+ )
766
+ class LlamaPreTrainedModel(PreTrainedModel):
767
+ config_class = MobileLLMConfig
768
+ base_model_prefix = "model"
769
+ supports_gradient_checkpointing = True
770
+ _no_split_modules = ["LlamaDecoderLayer"]
771
+ _skip_keys_device_placement = ["past_key_values"]
772
+ _supports_flash_attn_2 = True
773
+ _supports_sdpa = True
774
+ _supports_cache_class = True
775
+ _supports_quantized_cache = True
776
+ _supports_static_cache = True
777
+
778
+ def _init_weights(self, module):
779
+ std = self.config.initializer_range
780
+ if isinstance(module, nn.Linear):
781
+ module.weight.data.normal_(mean=0.0, std=std)
782
+ if module.bias is not None:
783
+ module.bias.data.zero_()
784
+ elif isinstance(module, nn.Embedding):
785
+ module.weight.data.normal_(mean=0.0, std=std)
786
+ if module.padding_idx is not None:
787
+ module.weight.data[module.padding_idx].zero_()
788
+
789
+
790
+ LLAMA_INPUTS_DOCSTRING = r"""
791
+ Args:
792
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
793
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
794
+ it.
795
+
796
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
797
+ [`PreTrainedTokenizer.__call__`] for details.
798
+
799
+ [What are input IDs?](../glossary#input-ids)
800
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
801
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
802
+
803
+ - 1 for tokens that are **not masked**,
804
+ - 0 for tokens that are **masked**.
805
+
806
+ [What are attention masks?](../glossary#attention-mask)
807
+
808
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
809
+ [`PreTrainedTokenizer.__call__`] for details.
810
+
811
+ If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
812
+ `past_key_values`).
813
+
814
+ If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
815
+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
816
+ information on the default strategy.
817
+
818
+ - 1 indicates the head is **not masked**,
819
+ - 0 indicates the head is **masked**.
820
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
821
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
822
+ config.n_positions - 1]`.
823
+
824
+ [What are position IDs?](../glossary#position-ids)
825
+ past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
826
+ Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
827
+ blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
828
+ returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
829
+
830
+ Two formats are allowed:
831
+ - a [`~cache_utils.Cache`] instance;
832
+ - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
833
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
834
+ cache format.
835
+
836
+ The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
837
+ legacy cache format will be returned.
838
+
839
+ If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
840
+ have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
841
+ of shape `(batch_size, sequence_length)`.
842
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
843
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
844
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
845
+ model's internal embedding lookup matrix.
846
+ use_cache (`bool`, *optional*):
847
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
848
+ `past_key_values`).
849
+ output_attentions (`bool`, *optional*):
850
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
851
+ tensors for more detail.
852
+ output_hidden_states (`bool`, *optional*):
853
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
854
+ more detail.
855
+ return_dict (`bool`, *optional*):
856
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
857
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
858
+ Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
859
+ this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
860
+ the complete sequence length.
861
+ """
862
+
863
+
864
+ @add_start_docstrings(
865
+ "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
866
+ LLAMA_START_DOCSTRING,
867
+ )
868
+ class MobileLLMModel(LlamaPreTrainedModel):
869
+ """
870
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`]
871
+
872
+ Args:
873
+ config: LlamaConfig
874
+ """
875
+
876
+ def __init__(self, config: MobileLLMConfig):
877
+ super().__init__(config)
878
+ self.padding_idx = config.pad_token_id
879
+ self.vocab_size = config.vocab_size
880
+
881
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
882
+ self.layers = nn.ModuleList(
883
+ [LlamaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
884
+ )
885
+ self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
886
+ self.gradient_checkpointing = False
887
+
888
+ self.layer_sharing = config.layer_sharing
889
+ # Initialize weights and apply final processing
890
+ self.post_init()
891
+
892
+ def get_input_embeddings(self):
893
+ return self.embed_tokens
894
+
895
+ def set_input_embeddings(self, value):
896
+ self.embed_tokens = value
897
+
898
+ @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
899
+ def forward(
900
+ self,
901
+ input_ids: torch.LongTensor = None,
902
+ attention_mask: Optional[torch.Tensor] = None,
903
+ position_ids: Optional[torch.LongTensor] = None,
904
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
905
+ inputs_embeds: Optional[torch.FloatTensor] = None,
906
+ use_cache: Optional[bool] = None,
907
+ output_attentions: Optional[bool] = None,
908
+ output_hidden_states: Optional[bool] = None,
909
+ return_dict: Optional[bool] = None,
910
+ cache_position: Optional[torch.LongTensor] = None,
911
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
912
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
913
+ output_hidden_states = (
914
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
915
+ )
916
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
917
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
918
+
919
+ if (input_ids is None) ^ (inputs_embeds is not None):
920
+ raise ValueError(
921
+ "You cannot specify both input_ids and inputs_embeds at the same time, and must specify either one"
922
+ )
923
+
924
+ if self.gradient_checkpointing and self.training and use_cache:
925
+ logger.warning_once(
926
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
927
+ )
928
+ use_cache = False
929
+
930
+ if inputs_embeds is None:
931
+ inputs_embeds = self.embed_tokens(input_ids)
932
+
933
+ return_legacy_cache = False
934
+ if use_cache and not isinstance(past_key_values, Cache): # kept for BC (non `Cache` `past_key_values` inputs)
935
+ return_legacy_cache = True
936
+ past_key_values = DynamicCache.from_legacy_cache(past_key_values)
937
+
938
+ if cache_position is None:
939
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
940
+ cache_position = torch.arange(
941
+ past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
942
+ )
943
+ if position_ids is None:
944
+ position_ids = cache_position.unsqueeze(0)
945
+
946
+ causal_mask = self._update_causal_mask(
947
+ attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
948
+ )
949
+
950
+ # embed positions
951
+ hidden_states = inputs_embeds
952
+
953
+ # decoder layers
954
+ all_hidden_states = () if output_hidden_states else None
955
+ all_self_attns = () if output_attentions else None
956
+ next_decoder_cache = None
957
+
958
+ for decoder_layer in self.layers:
959
+ if output_hidden_states:
960
+ all_hidden_states += (hidden_states,)
961
+
962
+ if self.gradient_checkpointing and self.training:
963
+ layer_outputs = self._gradient_checkpointing_func(
964
+ decoder_layer.__call__,
965
+ hidden_states,
966
+ causal_mask,
967
+ position_ids,
968
+ past_key_values,
969
+ output_attentions,
970
+ use_cache,
971
+ cache_position,
972
+ )
973
+ else:
974
+ layer_outputs = decoder_layer(
975
+ hidden_states,
976
+ attention_mask=causal_mask,
977
+ position_ids=position_ids,
978
+ past_key_value=past_key_values,
979
+ output_attentions=output_attentions,
980
+ use_cache=use_cache,
981
+ cache_position=cache_position,
982
+ )
983
+
984
+ hidden_states = layer_outputs[0]
985
+
986
+ if use_cache:
987
+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]
988
+
989
+ if output_attentions:
990
+ all_self_attns += (layer_outputs[1],)
991
+
992
+ # Repeat current layer if layer_sharing is enabled
993
+ if self.layer_sharing:
994
+ if output_hidden_states:
995
+ all_hidden_states += (hidden_states,)
996
+
997
+ if self.gradient_checkpointing and self.training:
998
+ layer_outputs = self._gradient_checkpointing_func(
999
+ decoder_layer.__call__,
1000
+ hidden_states,
1001
+ causal_mask,
1002
+ position_ids,
1003
+ past_key_values,
1004
+ output_attentions,
1005
+ use_cache,
1006
+ cache_position,
1007
+ )
1008
+ else:
1009
+ layer_outputs = decoder_layer(
1010
+ hidden_states,
1011
+ attention_mask=causal_mask,
1012
+ position_ids=position_ids,
1013
+ past_key_value=past_key_values,
1014
+ output_attentions=output_attentions,
1015
+ use_cache=use_cache,
1016
+ cache_position=cache_position,
1017
+ )
1018
+
1019
+ hidden_states = layer_outputs[0]
1020
+
1021
+ if use_cache:
1022
+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]
1023
+
1024
+ if output_attentions:
1025
+ all_self_attns += (layer_outputs[1],)
1026
+
1027
+ hidden_states = self.norm(hidden_states)
1028
+
1029
+ # add hidden states from the last decoder layer
1030
+ if output_hidden_states:
1031
+ all_hidden_states += (hidden_states,)
1032
+
1033
+ next_cache = next_decoder_cache if use_cache else None
1034
+ if return_legacy_cache:
1035
+ next_cache = next_cache.to_legacy_cache()
1036
+
1037
+ if not return_dict:
1038
+ return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
1039
+ return BaseModelOutputWithPast(
1040
+ last_hidden_state=hidden_states,
1041
+ past_key_values=next_cache,
1042
+ hidden_states=all_hidden_states,
1043
+ attentions=all_self_attns,
1044
+ )
1045
+
1046
+ def _update_causal_mask(
1047
+ self,
1048
+ attention_mask: torch.Tensor,
1049
+ input_tensor: torch.Tensor,
1050
+ cache_position: torch.Tensor,
1051
+ past_key_values: Cache,
1052
+ output_attentions: bool,
1053
+ ):
1054
+ # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static
1055
+ # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes.
1056
+ # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using
1057
+ # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114
1058
+
1059
+ if self.config._attn_implementation == "flash_attention_2":
1060
+ if attention_mask is not None and 0.0 in attention_mask:
1061
+ return attention_mask
1062
+ return None
1063
+
1064
+ # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
1065
+ # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
1066
+ # to infer the attention mask.
1067
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
1068
+ using_static_cache = isinstance(past_key_values, StaticCache)
1069
+
1070
+ # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
1071
+ if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions:
1072
+ if AttentionMaskConverter._ignore_causal_mask_sdpa(
1073
+ attention_mask,
1074
+ inputs_embeds=input_tensor,
1075
+ past_key_values_length=past_seen_tokens,
1076
+ is_training=self.training,
1077
+ ):
1078
+ return None
1079
+
1080
+ dtype, device = input_tensor.dtype, input_tensor.device
1081
+ min_dtype = torch.finfo(dtype).min
1082
+ sequence_length = input_tensor.shape[1]
1083
+ if using_static_cache:
1084
+ target_length = past_key_values.get_max_length()
1085
+ else:
1086
+ target_length = (
1087
+ attention_mask.shape[-1]
1088
+ if isinstance(attention_mask, torch.Tensor)
1089
+ else past_seen_tokens + sequence_length + 1
1090
+ )
1091
+
1092
+ if attention_mask is not None and attention_mask.dim() == 4:
1093
+ # in this case we assume that the mask comes already in inverted form and requires no inversion or slicing
1094
+ if attention_mask.max() != 0:
1095
+ raise ValueError("Custom 4D attention mask should be passed in inverted form with max==0`")
1096
+ causal_mask = attention_mask
1097
+ else:
1098
+ causal_mask = torch.full(
1099
+ (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device
1100
+ )
1101
+ if sequence_length != 1:
1102
+ causal_mask = torch.triu(causal_mask, diagonal=1)
1103
+ causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1)
1104
+ causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1)
1105
+ if attention_mask is not None:
1106
+ causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit
1107
+ mask_length = attention_mask.shape[-1]
1108
+ padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]
1109
+ padding_mask = padding_mask == 0
1110
+ causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(
1111
+ padding_mask, min_dtype
1112
+ )
1113
+ if (
1114
+ self.config._attn_implementation == "sdpa"
1115
+ and attention_mask is not None
1116
+ and attention_mask.device.type == "cuda"
1117
+ and not output_attentions
1118
+ ):
1119
+ # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
1120
+ # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
1121
+ # Details: https://github.com/pytorch/pytorch/issues/110213
1122
+ causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)
1123
+
1124
+ return causal_mask
1125
+
1126
+
1127
+ class MobileLLMForCausalLM(LlamaPreTrainedModel):
1128
+ _tied_weights_keys = ["lm_head.weight"]
1129
+
1130
+ def __init__(self, config):
1131
+ super().__init__(config)
1132
+ self.model = MobileLLMModel(config)
1133
+ self.vocab_size = config.vocab_size
1134
+ if not getattr(self.config, "share_embedding", False):
1135
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
1136
+
1137
+ # Initialize weights and apply final processing
1138
+ self.post_init()
1139
+
1140
+ def get_input_embeddings(self):
1141
+ return self.model.embed_tokens
1142
+
1143
+ def set_input_embeddings(self, value):
1144
+ self.model.embed_tokens = value
1145
+
1146
+ def get_output_embeddings(self):
1147
+ return (
1148
+ self.lm_head
1149
+ if not getattr(self.config, "share_embedding", False)
1150
+ else self.get_input_embeddings()
1151
+ )
1152
+
1153
+ def set_output_embeddings(self, new_embeddings):
1154
+ if not getattr(self.config, "share_embedding", False):
1155
+ self.lm_head = new_embeddings
1156
+ else:
1157
+ self.set_input_embeddings(new_embeddings)
1158
+
1159
+ def set_decoder(self, decoder):
1160
+ self.model = decoder
1161
+
1162
+ def get_decoder(self):
1163
+ return self.model
1164
+
1165
+ @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
1166
+ @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
1167
+ def forward(
1168
+ self,
1169
+ input_ids: torch.LongTensor = None,
1170
+ attention_mask: Optional[torch.Tensor] = None,
1171
+ position_ids: Optional[torch.LongTensor] = None,
1172
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
1173
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1174
+ labels: Optional[torch.LongTensor] = None,
1175
+ use_cache: Optional[bool] = None,
1176
+ output_attentions: Optional[bool] = None,
1177
+ output_hidden_states: Optional[bool] = None,
1178
+ return_dict: Optional[bool] = None,
1179
+ cache_position: Optional[torch.LongTensor] = None,
1180
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1181
+ r"""
1182
+ Args:
1183
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1184
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
1185
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
1186
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
1187
+
1188
+ Returns:
1189
+
1190
+ Example:
1191
+
1192
+ ```python
1193
+ >>> from transformers import AutoTokenizer, LlamaForCausalLM
1194
+
1195
+ >>> model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
1196
+ >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
1197
+
1198
+ >>> prompt = "Hey, are you conscious? Can you talk to me?"
1199
+ >>> inputs = tokenizer(prompt, return_tensors="pt")
1200
+
1201
+ >>> # Generate
1202
+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
1203
+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
1204
+ "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
1205
+ ```"""
1206
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1207
+ output_hidden_states = (
1208
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1209
+ )
1210
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1211
+
1212
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
1213
+ outputs = self.model(
1214
+ input_ids=input_ids,
1215
+ attention_mask=attention_mask,
1216
+ position_ids=position_ids,
1217
+ past_key_values=past_key_values,
1218
+ inputs_embeds=inputs_embeds,
1219
+ use_cache=use_cache,
1220
+ output_attentions=output_attentions,
1221
+ output_hidden_states=output_hidden_states,
1222
+ return_dict=return_dict,
1223
+ cache_position=cache_position,
1224
+ )
1225
+
1226
+ hidden_states = outputs[0]
1227
+ if self.config.pretraining_tp > 1:
1228
+ lm_head_slices = self.lm_head.weight.split(self.vocab_size // self.config.pretraining_tp, dim=0)
1229
+ logits = [F.linear(hidden_states, lm_head_slices[i]) for i in range(self.config.pretraining_tp)]
1230
+ logits = torch.cat(logits, dim=-1)
1231
+ else:
1232
+ if not getattr(self.config, "share_embedding", False):
1233
+ logits = self.lm_head(hidden_states)
1234
+ else:
1235
+ logits = F.linear(hidden_states, self.model.embed_tokens.weight)
1236
+ logits = logits.float()
1237
+
1238
+ loss = None
1239
+ if labels is not None:
1240
+ # Shift so that tokens < n predict n
1241
+ shift_logits = logits[..., :-1, :].contiguous()
1242
+ shift_labels = labels[..., 1:].contiguous()
1243
+ # Flatten the tokens
1244
+ loss_fct = CrossEntropyLoss()
1245
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
1246
+ shift_labels = shift_labels.view(-1)
1247
+ # Enable model parallelism
1248
+ shift_labels = shift_labels.to(shift_logits.device)
1249
+ loss = loss_fct(shift_logits, shift_labels)
1250
+
1251
+ if not return_dict:
1252
+ output = (logits,) + outputs[1:]
1253
+ return (loss,) + output if loss is not None else output
1254
+
1255
+ return CausalLMOutputWithPast(
1256
+ loss=loss,
1257
+ logits=logits,
1258
+ past_key_values=outputs.past_key_values,
1259
+ hidden_states=outputs.hidden_states,
1260
+ attentions=outputs.attentions,
1261
+ )
1262
+
1263
+ def prepare_inputs_for_generation(
1264
+ self,
1265
+ input_ids,
1266
+ past_key_values=None,
1267
+ attention_mask=None,
1268
+ inputs_embeds=None,
1269
+ cache_position=None,
1270
+ use_cache=True,
1271
+ **kwargs,
1272
+ ):
1273
+ past_length = 0
1274
+ if past_key_values is not None:
1275
+ # Past key values are always initialized with a `Cache` object -> no need for if-else anymore
1276
+ past_length = cache_position[0] if cache_position is not None else past_key_values.get_seq_length()
1277
+ max_cache_length = (
1278
+ torch.tensor(past_key_values.get_max_length(), device=input_ids.device)
1279
+ if past_key_values.get_max_length() is not None
1280
+ else None
1281
+ )
1282
+ cache_length = past_length if max_cache_length is None else torch.min(max_cache_length, past_length)
1283
+
1284
+ # Keep only the unprocessed tokens:
1285
+ # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
1286
+ # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as input)
1287
+ if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
1288
+ input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
1289
+ # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
1290
+ # input_ids based on the past_length.
1291
+ elif past_length < input_ids.shape[1]:
1292
+ input_ids = input_ids[:, past_length:]
1293
+ # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
1294
+
1295
+ # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
1296
+ if (
1297
+ max_cache_length is not None
1298
+ and attention_mask is not None
1299
+ and cache_length + input_ids.shape[1] > max_cache_length
1300
+ ):
1301
+ attention_mask = attention_mask[:, -max_cache_length:]
1302
+
1303
+ position_ids = kwargs.get("position_ids", None)
1304
+ if attention_mask is not None and position_ids is None:
1305
+ # create position_ids on the fly for batch generation
1306
+ position_ids = attention_mask.long().cumsum(-1) - 1
1307
+ position_ids.masked_fill_(attention_mask == 0, 1)
1308
+ if past_key_values:
1309
+ position_ids = position_ids[:, -input_ids.shape[1] :]
1310
+
1311
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
1312
+ if inputs_embeds is not None and past_length == 0:
1313
+ model_inputs = {"inputs_embeds": inputs_embeds}
1314
+ else:
1315
+ # The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise
1316
+ # recompiles graphs as the stride of the inputs is a guard. Ref: https://github.com/huggingface/transformers/pull/29114
1317
+ # TODO: use `next_tokens` directly instead.
1318
+ model_inputs = {"input_ids": input_ids.contiguous()}
1319
+
1320
+ input_length = position_ids.shape[-1] if position_ids is not None else input_ids.shape[-1]
1321
+ if cache_position is None:
1322
+ cache_position = torch.arange(past_length, past_length + input_length, device=input_ids.device)
1323
+ elif use_cache:
1324
+ cache_position = cache_position[-input_length:]
1325
+
1326
+ model_inputs.update(
1327
+ {
1328
+ "position_ids": position_ids,
1329
+ "cache_position": cache_position,
1330
+ "past_key_values": past_key_values,
1331
+ "use_cache": use_cache,
1332
+ "attention_mask": attention_mask,
1333
+ }
1334
+ )
1335
+ return model_inputs
1336
+
1337
+ @staticmethod
1338
+ def _reorder_cache(past_key_values, beam_idx):
1339
+ reordered_past = ()
1340
+ for layer_past in past_key_values:
1341
+ reordered_past += (
1342
+ tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
1343
+ )
1344
+ return reordered_past
1345
+
1346
+
1347
+ @add_start_docstrings(
1348
+ """
1349
+ The LLaMa Model transformer with a sequence classification head on top (linear layer).
1350
+
1351
+ [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models
1352
+ (e.g. GPT-2) do.
1353
+
1354
+ Since it does classification on the last token, it requires to know the position of the last token. If a
1355
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
1356
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
1357
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
1358
+ each row of the batch).
1359
+ """,
1360
+ LLAMA_START_DOCSTRING,
1361
+ )
1362
+ class MobileLLMForSequenceClassification(LlamaPreTrainedModel):
1363
+ def __init__(self, config):
1364
+ super().__init__(config)
1365
+ self.num_labels = config.num_labels
1366
+ self.model = MobileLLMModel(config)
1367
+ self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
1368
+
1369
+ # Initialize weights and apply final processing
1370
+ self.post_init()
1371
+
1372
+ def get_input_embeddings(self):
1373
+ return self.model.embed_tokens
1374
+
1375
+ def set_input_embeddings(self, value):
1376
+ self.model.embed_tokens = value
1377
+
1378
+ @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
1379
+ def forward(
1380
+ self,
1381
+ input_ids: torch.LongTensor = None,
1382
+ attention_mask: Optional[torch.Tensor] = None,
1383
+ position_ids: Optional[torch.LongTensor] = None,
1384
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
1385
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1386
+ labels: Optional[torch.LongTensor] = None,
1387
+ use_cache: Optional[bool] = None,
1388
+ output_attentions: Optional[bool] = None,
1389
+ output_hidden_states: Optional[bool] = None,
1390
+ return_dict: Optional[bool] = None,
1391
+ ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
1392
+ r"""
1393
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1394
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1395
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1396
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1397
+ """
1398
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1399
+
1400
+ transformer_outputs = self.model(
1401
+ input_ids,
1402
+ attention_mask=attention_mask,
1403
+ position_ids=position_ids,
1404
+ past_key_values=past_key_values,
1405
+ inputs_embeds=inputs_embeds,
1406
+ use_cache=use_cache,
1407
+ output_attentions=output_attentions,
1408
+ output_hidden_states=output_hidden_states,
1409
+ return_dict=return_dict,
1410
+ )
1411
+ hidden_states = transformer_outputs[0]
1412
+ logits = self.score(hidden_states)
1413
+
1414
+ if input_ids is not None:
1415
+ batch_size = input_ids.shape[0]
1416
+ else:
1417
+ batch_size = inputs_embeds.shape[0]
1418
+
1419
+ if self.config.pad_token_id is None and batch_size != 1:
1420
+ raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
1421
+ if self.config.pad_token_id is None:
1422
+ sequence_lengths = -1
1423
+ else:
1424
+ if input_ids is not None:
1425
+ # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
1426
+ sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
1427
+ sequence_lengths = sequence_lengths % input_ids.shape[-1]
1428
+ sequence_lengths = sequence_lengths.to(logits.device)
1429
+ else:
1430
+ sequence_lengths = -1
1431
+
1432
+ pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
1433
+
1434
+ loss = None
1435
+ if labels is not None:
1436
+ labels = labels.to(logits.device)
1437
+ if self.config.problem_type is None:
1438
+ if self.num_labels == 1:
1439
+ self.config.problem_type = "regression"
1440
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
1441
+ self.config.problem_type = "single_label_classification"
1442
+ else:
1443
+ self.config.problem_type = "multi_label_classification"
1444
+
1445
+ if self.config.problem_type == "regression":
1446
+ loss_fct = MSELoss()
1447
+ if self.num_labels == 1:
1448
+ loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
1449
+ else:
1450
+ loss = loss_fct(pooled_logits, labels)
1451
+ elif self.config.problem_type == "single_label_classification":
1452
+ loss_fct = CrossEntropyLoss()
1453
+ loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
1454
+ elif self.config.problem_type == "multi_label_classification":
1455
+ loss_fct = BCEWithLogitsLoss()
1456
+ loss = loss_fct(pooled_logits, labels)
1457
+ if not return_dict:
1458
+ output = (pooled_logits,) + transformer_outputs[1:]
1459
+ return ((loss,) + output) if loss is not None else output
1460
+
1461
+ return SequenceClassifierOutputWithPast(
1462
+ loss=loss,
1463
+ logits=pooled_logits,
1464
+ past_key_values=transformer_outputs.past_key_values,
1465
+ hidden_states=transformer_outputs.hidden_states,
1466
+ attentions=transformer_outputs.attentions,
1467
+ )
1468
+
1469
+
1470
+ @add_start_docstrings(
1471
+ """
1472
+ The Llama Model transformer with a span classification head on top for extractive question-answering tasks like
1473
+ SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`).
1474
+ """,
1475
+ LLAMA_START_DOCSTRING,
1476
+ )
1477
+ class MobileLLMForQuestionAnswering(LlamaPreTrainedModel):
1478
+ base_model_prefix = "transformer"
1479
+
1480
+ # Copied from transformers.models.bloom.modeling_bloom.BloomForQuestionAnswering.__init__ with Bloom->Llama
1481
+ def __init__(self, config):
1482
+ super().__init__(config)
1483
+ self.transformer = MobileLLMModel(config)
1484
+ self.qa_outputs = nn.Linear(config.hidden_size, 2)
1485
+
1486
+ # Initialize weights and apply final processing
1487
+ self.post_init()
1488
+
1489
+ def get_input_embeddings(self):
1490
+ return self.transformer.embed_tokens
1491
+
1492
+ def set_input_embeddings(self, value):
1493
+ self.transformer.embed_tokens = value
1494
+
1495
+ @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
1496
+ def forward(
1497
+ self,
1498
+ input_ids: Optional[torch.LongTensor] = None,
1499
+ attention_mask: Optional[torch.FloatTensor] = None,
1500
+ position_ids: Optional[torch.LongTensor] = None,
1501
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
1502
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1503
+ start_positions: Optional[torch.LongTensor] = None,
1504
+ end_positions: Optional[torch.LongTensor] = None,
1505
+ output_attentions: Optional[bool] = None,
1506
+ output_hidden_states: Optional[bool] = None,
1507
+ return_dict: Optional[bool] = None,
1508
+ ) -> Union[Tuple, QuestionAnsweringModelOutput]:
1509
+ r"""
1510
+ start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1511
+ Labels for position (index) of the start of the labelled span for computing the token classification loss.
1512
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
1513
+ are not taken into account for computing the loss.
1514
+ end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1515
+ Labels for position (index) of the end of the labelled span for computing the token classification loss.
1516
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
1517
+ are not taken into account for computing the loss.
1518
+ """
1519
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1520
+
1521
+ outputs = self.transformer(
1522
+ input_ids,
1523
+ attention_mask=attention_mask,
1524
+ position_ids=position_ids,
1525
+ past_key_values=past_key_values,
1526
+ inputs_embeds=inputs_embeds,
1527
+ output_attentions=output_attentions,
1528
+ output_hidden_states=output_hidden_states,
1529
+ return_dict=return_dict,
1530
+ )
1531
+
1532
+ sequence_output = outputs[0]
1533
+
1534
+ logits = self.qa_outputs(sequence_output)
1535
+ start_logits, end_logits = logits.split(1, dim=-1)
1536
+ start_logits = start_logits.squeeze(-1).contiguous()
1537
+ end_logits = end_logits.squeeze(-1).contiguous()
1538
+
1539
+ total_loss = None
1540
+ if start_positions is not None and end_positions is not None:
1541
+ # If we are on multi-GPU, split add a dimension
1542
+ if len(start_positions.size()) > 1:
1543
+ start_positions = start_positions.squeeze(-1).to(start_logits.device)
1544
+ if len(end_positions.size()) > 1:
1545
+ end_positions = end_positions.squeeze(-1).to(end_logits.device)
1546
+ # sometimes the start/end positions are outside our model inputs, we ignore these terms
1547
+ ignored_index = start_logits.size(1)
1548
+ start_positions = start_positions.clamp(0, ignored_index)
1549
+ end_positions = end_positions.clamp(0, ignored_index)
1550
+
1551
+ loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
1552
+ start_loss = loss_fct(start_logits, start_positions)
1553
+ end_loss = loss_fct(end_logits, end_positions)
1554
+ total_loss = (start_loss + end_loss) / 2
1555
+
1556
+ if not return_dict:
1557
+ output = (start_logits, end_logits) + outputs[2:]
1558
+ return ((total_loss,) + output) if total_loss is not None else output
1559
+
1560
+ return QuestionAnsweringModelOutput(
1561
+ loss=total_loss,
1562
+ start_logits=start_logits,
1563
+ end_logits=end_logits,
1564
+ hidden_states=outputs.hidden_states,
1565
+ attentions=outputs.attentions,
1566
+ )
1567
+
1568
+
1569
+ @add_start_docstrings(
1570
+ """
1571
+ The Llama Model transformer with a token classification head on top (a linear layer on top of the hidden-states
1572
+ output) e.g. for Named-Entity-Recognition (NER) tasks.
1573
+ """,
1574
+ LLAMA_START_DOCSTRING,
1575
+ )
1576
+ class MobileLLMForTokenClassification(LlamaPreTrainedModel):
1577
+ def __init__(self, config):
1578
+ super().__init__(config)
1579
+ self.num_labels = config.num_labels
1580
+ self.model = MobileLLMModel(config)
1581
+ if getattr(config, "classifier_dropout", None) is not None:
1582
+ classifier_dropout = config.classifier_dropout
1583
+ elif getattr(config, "hidden_dropout", None) is not None:
1584
+ classifier_dropout = config.hidden_dropout
1585
+ else:
1586
+ classifier_dropout = 0.1
1587
+ self.dropout = nn.Dropout(classifier_dropout)
1588
+ self.score = nn.Linear(config.hidden_size, config.num_labels)
1589
+
1590
+ # Initialize weights and apply final processing
1591
+ self.post_init()
1592
+
1593
+ def get_input_embeddings(self):
1594
+ return self.model.embed_tokens
1595
+
1596
+ def set_input_embeddings(self, value):
1597
+ self.model.embed_tokens = value
1598
+
1599
+ @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
1600
+ def forward(
1601
+ self,
1602
+ input_ids: Optional[torch.LongTensor] = None,
1603
+ attention_mask: Optional[torch.Tensor] = None,
1604
+ position_ids: Optional[torch.LongTensor] = None,
1605
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
1606
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1607
+ labels: Optional[torch.LongTensor] = None,
1608
+ use_cache: Optional[bool] = None,
1609
+ output_attentions: Optional[bool] = None,
1610
+ output_hidden_states: Optional[bool] = None,
1611
+ return_dict: Optional[bool] = None,
1612
+ ) -> Union[Tuple, TokenClassifierOutput]:
1613
+ r"""
1614
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1615
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1616
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1617
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1618
+ """
1619
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1620
+
1621
+ outputs = self.model(
1622
+ input_ids,
1623
+ attention_mask=attention_mask,
1624
+ position_ids=position_ids,
1625
+ past_key_values=past_key_values,
1626
+ inputs_embeds=inputs_embeds,
1627
+ use_cache=use_cache,
1628
+ output_attentions=output_attentions,
1629
+ output_hidden_states=output_hidden_states,
1630
+ return_dict=return_dict,
1631
+ )
1632
+ sequence_output = outputs[0]
1633
+ sequence_output = self.dropout(sequence_output)
1634
+ logits = self.score(sequence_output)
1635
+
1636
+ loss = None
1637
+ if labels is not None:
1638
+ loss_fct = CrossEntropyLoss()
1639
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
1640
+
1641
+ if not return_dict:
1642
+ output = (logits,) + outputs[2:]
1643
+ return ((loss,) + output) if loss is not None else output
1644
+
1645
+ return TokenClassifierOutput(
1646
+ loss=loss,
1647
+ logits=logits,
1648
+ hidden_states=outputs.hidden_states,
1649
+ attentions=outputs.attentions,
1650
+ )
1651
+
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:962565fa7d9fb2934d7f1a9e2a677293e085445978c0fbbd1c7efffdd5840a33
3
+ size 690804562
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "", "eos_token": "", "model_max_length": 1000000000000000019884624838656, "tokenizer_class": "LlamaTokenizer", "unk_token": ""}