TRI-ML
/

DCLM-1B-v0

sedrickkeh commited on Jul 18

Commit

f749ecd

•

1 Parent(s): b2b37c9

Update config.json

Files changed (1) hide show

config.json CHANGED Viewed

@@ -12,7 +12,7 @@
     "dim": 2048,
     "ffn_type": "swiglu_torch",
     "model": "open_lm_1b_swiglutorch",
-    "model_norm": "gain_only_lp_layer_norm",
     "moe_capacity_factor": 1.25,
     "moe_expert_model_parallelism": false,
     "moe_freq": 0,

     "dim": 2048,
     "ffn_type": "swiglu_torch",
     "model": "open_lm_1b_swiglutorch",
+    "norm_type": "gain_only_lp_layer_norm",
     "moe_capacity_factor": 1.25,
     "moe_expert_model_parallelism": false,
     "moe_freq": 0,