{ "_name_or_path": "checkpoints/mtgv/MobileLLaMA-1.4B-Chat", "anyprec": { "arch_config": { "layers_name": "layers", "model_name": "model", "module_names": [ "self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj" ] }, "group_count": 1, "parent_precision": 4, "seed_precision": 2, "sparse_numvals": { "model.layers.0.mlp.down_proj": 191326, "model.layers.0.mlp.gate_proj": 189994, "model.layers.0.mlp.up_proj": 180517, "model.layers.0.self_attn.k_proj": 176786, "model.layers.0.self_attn.o_proj": 72994, "model.layers.0.self_attn.q_proj": 158594, "model.layers.0.self_attn.v_proj": 81335, "model.layers.1.mlp.down_proj": 185829, "model.layers.1.mlp.gate_proj": 191585, "model.layers.1.mlp.up_proj": 180912, "model.layers.1.self_attn.k_proj": 355737, "model.layers.1.self_attn.o_proj": 125591, "model.layers.1.self_attn.q_proj": 351572, "model.layers.1.self_attn.v_proj": 92291, "model.layers.10.mlp.down_proj": 185125, "model.layers.10.mlp.gate_proj": 212731, "model.layers.10.mlp.up_proj": 191684, "model.layers.10.self_attn.k_proj": 192285, "model.layers.10.self_attn.o_proj": 85016, "model.layers.10.self_attn.q_proj": 176785, "model.layers.10.self_attn.v_proj": 88862, "model.layers.11.mlp.down_proj": 187377, "model.layers.11.mlp.gate_proj": 214387, "model.layers.11.mlp.up_proj": 193449, "model.layers.11.self_attn.k_proj": 183041, "model.layers.11.self_attn.o_proj": 79355, "model.layers.11.self_attn.q_proj": 179551, "model.layers.11.self_attn.v_proj": 85991, "model.layers.12.mlp.down_proj": 193165, "model.layers.12.mlp.gate_proj": 228979, "model.layers.12.mlp.up_proj": 202530, "model.layers.12.self_attn.k_proj": 168033, "model.layers.12.self_attn.o_proj": 74731, "model.layers.12.self_attn.q_proj": 159784, "model.layers.12.self_attn.v_proj": 85258, "model.layers.13.mlp.down_proj": 199578, "model.layers.13.mlp.gate_proj": 254937, "model.layers.13.mlp.up_proj": 205412, "model.layers.13.self_attn.k_proj": 168574, "model.layers.13.self_attn.o_proj": 80155, "model.layers.13.self_attn.q_proj": 166589, "model.layers.13.self_attn.v_proj": 96907, "model.layers.14.mlp.down_proj": 206123, "model.layers.14.mlp.gate_proj": 267261, "model.layers.14.mlp.up_proj": 213029, "model.layers.14.self_attn.k_proj": 175625, "model.layers.14.self_attn.o_proj": 79709, "model.layers.14.self_attn.q_proj": 165058, "model.layers.14.self_attn.v_proj": 92779, "model.layers.15.mlp.down_proj": 202519, "model.layers.15.mlp.gate_proj": 259676, "model.layers.15.mlp.up_proj": 213468, "model.layers.15.self_attn.k_proj": 178037, "model.layers.15.self_attn.o_proj": 82764, "model.layers.15.self_attn.q_proj": 178982, "model.layers.15.self_attn.v_proj": 95999, "model.layers.16.mlp.down_proj": 201854, "model.layers.16.mlp.gate_proj": 250550, "model.layers.16.mlp.up_proj": 213411, "model.layers.16.self_attn.k_proj": 160056, "model.layers.16.self_attn.o_proj": 83224, "model.layers.16.self_attn.q_proj": 157716, "model.layers.16.self_attn.v_proj": 92808, "model.layers.17.mlp.down_proj": 198730, "model.layers.17.mlp.gate_proj": 237767, "model.layers.17.mlp.up_proj": 209909, "model.layers.17.self_attn.k_proj": 171644, "model.layers.17.self_attn.o_proj": 83754, "model.layers.17.self_attn.q_proj": 172707, "model.layers.17.self_attn.v_proj": 90734, "model.layers.18.mlp.down_proj": 192702, "model.layers.18.mlp.gate_proj": 222867, "model.layers.18.mlp.up_proj": 206730, "model.layers.18.self_attn.k_proj": 156690, "model.layers.18.self_attn.o_proj": 84609, "model.layers.18.self_attn.q_proj": 174408, "model.layers.18.self_attn.v_proj": 89341, "model.layers.19.mlp.down_proj": 191205, "model.layers.19.mlp.gate_proj": 213948, "model.layers.19.mlp.up_proj": 201900, "model.layers.19.self_attn.k_proj": 155221, "model.layers.19.self_attn.o_proj": 100111, "model.layers.19.self_attn.q_proj": 154902, "model.layers.19.self_attn.v_proj": 102790, "model.layers.2.mlp.down_proj": 180930, "model.layers.2.mlp.gate_proj": 178939, "model.layers.2.mlp.up_proj": 178176, "model.layers.2.self_attn.k_proj": 254302, "model.layers.2.self_attn.o_proj": 85627, "model.layers.2.self_attn.q_proj": 208855, "model.layers.2.self_attn.v_proj": 78918, "model.layers.20.mlp.down_proj": 189531, "model.layers.20.mlp.gate_proj": 208030, "model.layers.20.mlp.up_proj": 200150, "model.layers.20.self_attn.k_proj": 136537, "model.layers.20.self_attn.o_proj": 91412, "model.layers.20.self_attn.q_proj": 136465, "model.layers.20.self_attn.v_proj": 87525, "model.layers.21.mlp.down_proj": 189005, "model.layers.21.mlp.gate_proj": 197507, "model.layers.21.mlp.up_proj": 192699, "model.layers.21.self_attn.k_proj": 140343, "model.layers.21.self_attn.o_proj": 117585, "model.layers.21.self_attn.q_proj": 134785, "model.layers.21.self_attn.v_proj": 111843, "model.layers.22.mlp.down_proj": 204005, "model.layers.22.mlp.gate_proj": 210880, "model.layers.22.mlp.up_proj": 200959, "model.layers.22.self_attn.k_proj": 150879, "model.layers.22.self_attn.o_proj": 101330, "model.layers.22.self_attn.q_proj": 151577, "model.layers.22.self_attn.v_proj": 98921, "model.layers.23.mlp.down_proj": 277332, "model.layers.23.mlp.gate_proj": 245277, "model.layers.23.mlp.up_proj": 258698, "model.layers.23.self_attn.k_proj": 118274, "model.layers.23.self_attn.o_proj": 114643, "model.layers.23.self_attn.q_proj": 113038, "model.layers.23.self_attn.v_proj": 107870, "model.layers.3.mlp.down_proj": 178845, "model.layers.3.mlp.gate_proj": 177565, "model.layers.3.mlp.up_proj": 178331, "model.layers.3.self_attn.k_proj": 187462, "model.layers.3.self_attn.o_proj": 70922, "model.layers.3.self_attn.q_proj": 152011, "model.layers.3.self_attn.v_proj": 72948, "model.layers.4.mlp.down_proj": 183192, "model.layers.4.mlp.gate_proj": 181505, "model.layers.4.mlp.up_proj": 180662, "model.layers.4.self_attn.k_proj": 186191, "model.layers.4.self_attn.o_proj": 68503, "model.layers.4.self_attn.q_proj": 143279, "model.layers.4.self_attn.v_proj": 71540, "model.layers.5.mlp.down_proj": 182326, "model.layers.5.mlp.gate_proj": 180629, "model.layers.5.mlp.up_proj": 180722, "model.layers.5.self_attn.k_proj": 179644, "model.layers.5.self_attn.o_proj": 78534, "model.layers.5.self_attn.q_proj": 155296, "model.layers.5.self_attn.v_proj": 84819, "model.layers.6.mlp.down_proj": 179180, "model.layers.6.mlp.gate_proj": 187118, "model.layers.6.mlp.up_proj": 179770, "model.layers.6.self_attn.k_proj": 181747, "model.layers.6.self_attn.o_proj": 86711, "model.layers.6.self_attn.q_proj": 156780, "model.layers.6.self_attn.v_proj": 97342, "model.layers.7.mlp.down_proj": 179455, "model.layers.7.mlp.gate_proj": 190350, "model.layers.7.mlp.up_proj": 185410, "model.layers.7.self_attn.k_proj": 152715, "model.layers.7.self_attn.o_proj": 75703, "model.layers.7.self_attn.q_proj": 130349, "model.layers.7.self_attn.v_proj": 81380, "model.layers.8.mlp.down_proj": 184587, "model.layers.8.mlp.gate_proj": 206660, "model.layers.8.mlp.up_proj": 192351, "model.layers.8.self_attn.k_proj": 167562, "model.layers.8.self_attn.o_proj": 90781, "model.layers.8.self_attn.q_proj": 142073, "model.layers.8.self_attn.v_proj": 103015, "model.layers.9.mlp.down_proj": 183099, "model.layers.9.mlp.gate_proj": 202416, "model.layers.9.mlp.up_proj": 192308, "model.layers.9.self_attn.k_proj": 184713, "model.layers.9.self_attn.o_proj": 92257, "model.layers.9.self_attn.q_proj": 170487, "model.layers.9.self_attn.v_proj": 101971 } }, "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 5632, "max_position_embeddings": 2048, "max_sequence_length": 2048, "model_type": "llama", "num_attention_heads": 16, "num_hidden_layers": 24, "num_key_value_heads": 16, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000.0, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.39.3", "use_cache": true, "vocab_size": 32000 }