Update configuration_Llamoe.py

configuration_Llamoe.py

@@ -21,7 +21,7 @@ class LlamoeConfig(PretrainedConfig):
         num_hidden_layers=32,
         num_attention_heads=32,
         num_key_value_heads=32,
-        head_dim=256,
+        head_dim=128,
         hidden_act="silu",
         max_position_embeddings=2048,
         initializer_range=0.02,