Update modeling_Llamoe.py
Browse files- modeling_Llamoe.py +1 -1
modeling_Llamoe.py
CHANGED
@@ -251,7 +251,7 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
|
251 |
|
252 |
|
253 |
class LlamoeBlockSparseTop2MLP(nn.Module):
|
254 |
-
def __init__(self, config:
|
255 |
super().__init__()
|
256 |
self.ffn_dim = config.intermediate_size
|
257 |
self.hidden_dim = config.hidden_size
|
|
|
251 |
|
252 |
|
253 |
class LlamoeBlockSparseTop2MLP(nn.Module):
|
254 |
+
def __init__(self, config: LlamoeConfig):
|
255 |
super().__init__()
|
256 |
self.ffn_dim = config.intermediate_size
|
257 |
self.hidden_dim = config.hidden_size
|