File size: 858 Bytes
8afc994 29da0e7 8afc994 ec422f2 8afc994 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
{
"architectures": [
"OpenLMModel"
],
"model_type": "openlm",
"params": null,
"apply_qk_norm": true,
"attn_activation": null,
"attn_name": "torch_attn",
"attn_seq_scalar": null,
"attn_seq_scalar_alpha": null,
"dim": 4096,
"ffn_type": "swiglu",
"model": "open_lm_7b",
"norm_type": "gain_only_lp_layer_norm",
"moe_capacity_factor": 1.25,
"moe_expert_model_parallelism": false,
"moe_freq": 0,
"moe_loss_weight": 0.1,
"moe_num_experts": null,
"moe_top_k": 2,
"moe_weight_parallelism": false,
"n_heads": 32,
"n_layers": 32,
"norm_eps": 1e-05,
"positional_embedding_type": "rotary",
"post_embed_norm": false,
"qk_norm": true,
"seq_len": 2048,
"vocab_size": 50432,
"weight_tying": false,
"torch_dtype": "float32",
"transformers_version": "4.38.2"
}
|