Update modeling_Llamoe.py
Browse files- modeling_Llamoe.py +1 -1
modeling_Llamoe.py
CHANGED
@@ -525,7 +525,7 @@ class LlamoeFlashAttention2(LlamoeAttention):
|
|
525 |
|
526 |
|
527 |
# Copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Gemmoe
|
528 |
-
class LlamoeSdpaAttention(
|
529 |
"""
|
530 |
Gemmoe attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
|
531 |
`GemmoeAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
|
|
|
525 |
|
526 |
|
527 |
# Copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Gemmoe
|
528 |
+
class LlamoeSdpaAttention(LlamoeAttention):
|
529 |
"""
|
530 |
Gemmoe attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
|
531 |
`GemmoeAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
|