diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" --- "a/ndarray-cache.json" +++ "b/ndarray-cache.json" @@ -2,7 +2,7 @@ "metadata": { "ParamSize": 198, "ParamBytes": 3087428608.0, - "BitsPerParam": 16.0 + "BitsPerParam": 13.89882147873375 }, "records": [ { @@ -22,7 +22,7 @@ "byteOffset": 0 } ], - "md5sum": "4004155c68c07080f39231f289b9c9d3" + "md5sum": "fac6ce8ba0d8ae438d10b5bb546c7cbd" }, { "dataPath": "params_shard_1.bin", @@ -41,64 +41,15 @@ "byteOffset": 0 } ], - "md5sum": "8d9c6137ab1639c5864a3e4f0d47a945" + "md5sum": "b0c672ae0f591da2d49f4dce650e72ae" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", - "nbytes": 27535360, - "records": [ - { - "name": "model.layers.0.input_layernorm.weight", - "shape": [ - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 0 - }, - { - "name": "model.layers.0.mlp.down_proj.weight", - "shape": [ - 1536, - 8960 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 27525120, - "byteOffset": 3072 - }, - { - "name": "model.layers.0.post_attention_layernorm.weight", - "shape": [ - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 27528192 - }, - { - "name": "model.layers.0.self_attn.c_attn.bias", - "shape": [ - 2048 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 27531264 - } - ], - "md5sum": "9fd8f6531c7b68276a0e21513cf4d24c" - }, - { - "dataPath": "params_shard_3.bin", - "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.1.mlp.down_proj.weight", + "name": "model.layers.0.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -109,10 +60,10 @@ "byteOffset": 0 } ], - "md5sum": "db7e25bc89a29b9b7bbf6714baa3dc24" + "md5sum": "db1e935ff9f5b6ff5da80f8b6d1628ff" }, { - "dataPath": "params_shard_4.bin", + "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ @@ -128,15 +79,15 @@ "byteOffset": 0 } ], - "md5sum": "13cf30b3f20ad035712c136f18f7d38d" + "md5sum": "baee6aadc8be0f23324126937a4cf15b" }, { - "dataPath": "params_shard_5.bin", + "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.10.mlp.down_proj.weight", + "name": "model.layers.1.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -147,15 +98,15 @@ "byteOffset": 0 } ], - "md5sum": "dfd4e0fba5875fd1d7db311377b87fe5" + "md5sum": "9581d907a6edac4b6bc1c357d0d2343f" }, { - "dataPath": "params_shard_6.bin", + "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.10.mlp.gate_up_proj.weight", + "name": "model.layers.2.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -166,15 +117,15 @@ "byteOffset": 0 } ], - "md5sum": "d3c8c121be697e30d9da502d459908dc" + "md5sum": "a8b915076ac9a72930fb6871e62d46ef" }, { - "dataPath": "params_shard_7.bin", + "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.11.mlp.down_proj.weight", + "name": "model.layers.2.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -185,29 +136,10 @@ "byteOffset": 0 } ], - "md5sum": "c303778173a91bfdc754b414416c5160" - }, - { - "dataPath": "params_shard_8.bin", - "format": "raw-shard", - "nbytes": 55050240, - "records": [ - { - "name": "model.layers.11.mlp.gate_up_proj.weight", - "shape": [ - 17920, - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 55050240, - "byteOffset": 0 - } - ], - "md5sum": "ab163c3a0afabe864dc1dfaeae1a9908" + "md5sum": "972150741a85989c34825d9c970f6741" }, { - "dataPath": "params_shard_9.bin", + "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ @@ -223,45 +155,45 @@ "byteOffset": 0 }, { - "name": "model.layers.0.self_attn.o_proj.weight", + "name": "model.layers.0.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.1.input_layernorm.weight", + "name": "model.layers.0.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.1.post_attention_layernorm.weight", + "name": "model.layers.0.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.1.self_attn.c_attn.bias", + "name": "model.layers.0.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { "name": "model.layers.1.self_attn.c_attn.weight", @@ -275,48 +207,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.1.self_attn.o_proj.weight", + "name": "model.layers.1.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.10.input_layernorm.weight", + "name": "model.layers.1.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.10.post_attention_layernorm.weight", + "name": "model.layers.1.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.10.self_attn.c_attn.bias", + "name": "model.layers.1.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.10.self_attn.c_attn.weight", + "name": "model.layers.2.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -327,7 +259,17 @@ "byteOffset": 22040576 }, { - "name": "model.layers.10.self_attn.o_proj.weight", + "name": "model.layers.2.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 28332032 + }, + { + "name": "model.layers.2.self_attn.o_proj.weight", "shape": [ 1536, 1536 @@ -335,48 +277,57 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, - "byteOffset": 28332032 + "byteOffset": 28336128 }, { - "name": "model.layers.11.input_layernorm.weight", + "name": "model.layers.2.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33050624 + "byteOffset": 33054720 }, { - "name": "model.layers.11.post_attention_layernorm.weight", + "name": "model.layers.2.post_attention_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 - }, + "byteOffset": 33057792 + } + ], + "md5sum": "cc217b23db4df89fcfed2ffa54ee328a" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 55050240, + "records": [ { - "name": "model.layers.11.self_attn.c_attn.bias", + "name": "model.layers.3.mlp.gate_up_proj.weight", "shape": [ - 2048 + 17920, + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 55050240, + "byteOffset": 0 } ], - "md5sum": "2e9b51ed2ac487ad15028dc401beeb7d" + "md5sum": "969f8fa80cdf5b272c2e607a2119c4e5" }, { - "dataPath": "params_shard_10.bin", + "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.12.mlp.down_proj.weight", + "name": "model.layers.3.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -387,15 +338,15 @@ "byteOffset": 0 } ], - "md5sum": "cc304778043000c0f442d99b072b593c" + "md5sum": "d860175d6e9380a30cfbe06262104ec7" }, { - "dataPath": "params_shard_11.bin", + "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.12.mlp.gate_up_proj.weight", + "name": "model.layers.4.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -406,15 +357,15 @@ "byteOffset": 0 } ], - "md5sum": "26700b8899f4d41d92f8439f3e0f1c28" + "md5sum": "3f5dad1ba616506c4e5084bb18bbf1d6" }, { - "dataPath": "params_shard_12.bin", + "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.13.mlp.down_proj.weight", + "name": "model.layers.4.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -425,15 +376,15 @@ "byteOffset": 0 } ], - "md5sum": "2ddac3acda693d7656dc82d1cb44a5fa" + "md5sum": "b993af8733b274d5e5aaacdc6462f74f" }, { - "dataPath": "params_shard_13.bin", + "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.13.mlp.gate_up_proj.weight", + "name": "model.layers.5.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -444,15 +395,15 @@ "byteOffset": 0 } ], - "md5sum": "00477d2aa802e741a31ae46a2af61519" + "md5sum": "2877a35f91a04b1c58f6a824e8d61e87" }, { - "dataPath": "params_shard_14.bin", + "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.14.mlp.down_proj.weight", + "name": "model.layers.5.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -463,34 +414,15 @@ "byteOffset": 0 } ], - "md5sum": "a0eb921170b36a016d3b8656caa53b8d" - }, - { - "dataPath": "params_shard_15.bin", - "format": "raw-shard", - "nbytes": 55050240, - "records": [ - { - "name": "model.layers.14.mlp.gate_up_proj.weight", - "shape": [ - 17920, - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 55050240, - "byteOffset": 0 - } - ], - "md5sum": "d80214adb8ccfa0343cf8adb0caec5a4" + "md5sum": "8bad654a3c485e170a6f0e5ddab81db0" }, { - "dataPath": "params_shard_16.bin", + "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.11.self_attn.c_attn.weight", + "name": "model.layers.3.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -501,48 +433,48 @@ "byteOffset": 0 }, { - "name": "model.layers.11.self_attn.o_proj.weight", + "name": "model.layers.3.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.12.input_layernorm.weight", + "name": "model.layers.3.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.12.post_attention_layernorm.weight", + "name": "model.layers.3.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.12.self_attn.c_attn.bias", + "name": "model.layers.3.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.12.self_attn.c_attn.weight", + "name": "model.layers.4.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -553,48 +485,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.12.self_attn.o_proj.weight", + "name": "model.layers.4.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.13.input_layernorm.weight", + "name": "model.layers.4.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.13.post_attention_layernorm.weight", + "name": "model.layers.4.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.13.self_attn.c_attn.bias", + "name": "model.layers.4.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.13.self_attn.c_attn.weight", + "name": "model.layers.5.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -605,75 +537,56 @@ "byteOffset": 22040576 }, { - "name": "model.layers.13.self_attn.o_proj.weight", + "name": "model.layers.5.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 28332032 }, { - "name": "model.layers.14.input_layernorm.weight", + "name": "model.layers.5.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 33050624 + "nbytes": 4718592, + "byteOffset": 28336128 }, { - "name": "model.layers.14.post_attention_layernorm.weight", + "name": "model.layers.5.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 + "byteOffset": 33054720 }, { - "name": "model.layers.14.self_attn.c_attn.bias", - "shape": [ - 2048 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 - } - ], - "md5sum": "836ad76f4bf6b3ea6142a57d73863eab" - }, - { - "dataPath": "params_shard_17.bin", - "format": "raw-shard", - "nbytes": 27525120, - "records": [ - { - "name": "model.layers.15.mlp.down_proj.weight", + "name": "model.layers.5.post_attention_layernorm.weight", "shape": [ - 1536, - 8960 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 27525120, - "byteOffset": 0 + "nbytes": 3072, + "byteOffset": 33057792 } ], - "md5sum": "69c8ebf98b69e0a7bed86b1b095d4e58" + "md5sum": "9a67ac0bf8178e4054e781eb27f403ea" }, { - "dataPath": "params_shard_18.bin", + "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.15.mlp.gate_up_proj.weight", + "name": "model.layers.6.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -684,15 +597,15 @@ "byteOffset": 0 } ], - "md5sum": "bc5b5e85e1e1ca4a340dc0ae73d0cf70" + "md5sum": "ccd5502649f7168e48df2ccbaeeb1d9b" }, { - "dataPath": "params_shard_19.bin", + "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.16.mlp.down_proj.weight", + "name": "model.layers.6.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -703,15 +616,15 @@ "byteOffset": 0 } ], - "md5sum": "1f0d7a92b72e8c084a47173269fabe37" + "md5sum": "271cf1aad384c2d628d1aaf62a140a9c" }, { - "dataPath": "params_shard_20.bin", + "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.16.mlp.gate_up_proj.weight", + "name": "model.layers.7.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -722,15 +635,15 @@ "byteOffset": 0 } ], - "md5sum": "7868afb9efef145589b5d006eaa1d2cb" + "md5sum": "901c1912a0418111b928559f750e0d95" }, { - "dataPath": "params_shard_21.bin", + "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.17.mlp.down_proj.weight", + "name": "model.layers.7.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -741,15 +654,15 @@ "byteOffset": 0 } ], - "md5sum": "64dd36a133277b89ecb7b753fa45a053" + "md5sum": "379946453685b3ee0b55935c9538eecd" }, { - "dataPath": "params_shard_22.bin", + "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.17.mlp.gate_up_proj.weight", + "name": "model.layers.8.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -760,15 +673,34 @@ "byteOffset": 0 } ], - "md5sum": "9ed20f54aebad5411cce28791db1bd4f" + "md5sum": "56fa2871d8315eef920e5df9970ea9fe" }, { - "dataPath": "params_shard_23.bin", + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 27525120, + "records": [ + { + "name": "model.layers.8.mlp.down_proj.weight", + "shape": [ + 1536, + 8960 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 27525120, + "byteOffset": 0 + } + ], + "md5sum": "817b9f5a7e095c5f05876994890301b4" + }, + { + "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.14.self_attn.c_attn.weight", + "name": "model.layers.6.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -779,48 +711,48 @@ "byteOffset": 0 }, { - "name": "model.layers.14.self_attn.o_proj.weight", + "name": "model.layers.6.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.15.input_layernorm.weight", + "name": "model.layers.6.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.15.post_attention_layernorm.weight", + "name": "model.layers.6.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.15.self_attn.c_attn.bias", + "name": "model.layers.6.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.15.self_attn.c_attn.weight", + "name": "model.layers.7.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -831,48 +763,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.15.self_attn.o_proj.weight", + "name": "model.layers.7.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.16.input_layernorm.weight", + "name": "model.layers.7.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.16.post_attention_layernorm.weight", + "name": "model.layers.7.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.16.self_attn.c_attn.bias", + "name": "model.layers.7.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.16.self_attn.c_attn.weight", + "name": "model.layers.8.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -883,7 +815,17 @@ "byteOffset": 22040576 }, { - "name": "model.layers.16.self_attn.o_proj.weight", + "name": "model.layers.8.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 28332032 + }, + { + "name": "model.layers.8.self_attn.o_proj.weight", "shape": [ 1536, 1536 @@ -891,48 +833,57 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, - "byteOffset": 28332032 + "byteOffset": 28336128 }, { - "name": "model.layers.17.input_layernorm.weight", + "name": "model.layers.8.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33050624 + "byteOffset": 33054720 }, { - "name": "model.layers.17.post_attention_layernorm.weight", + "name": "model.layers.8.post_attention_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 - }, + "byteOffset": 33057792 + } + ], + "md5sum": "bcd5c8951311cb8debd10c7a9cbd2764" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 55050240, + "records": [ { - "name": "model.layers.17.self_attn.c_attn.bias", + "name": "model.layers.9.mlp.gate_up_proj.weight", "shape": [ - 2048 + 17920, + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 55050240, + "byteOffset": 0 } ], - "md5sum": "d3ca8d87d2da5c44ba5209a2791736da" + "md5sum": "faf5cbdd3c2ff11883ce9f8554bcf498" }, { - "dataPath": "params_shard_24.bin", + "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.18.mlp.down_proj.weight", + "name": "model.layers.9.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -943,15 +894,15 @@ "byteOffset": 0 } ], - "md5sum": "3dd39a6d6ad6ed3ea7b684810336118a" + "md5sum": "17ae02bef8401386b5eb9f653b2f420a" }, { - "dataPath": "params_shard_25.bin", + "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.18.mlp.gate_up_proj.weight", + "name": "model.layers.10.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -962,15 +913,15 @@ "byteOffset": 0 } ], - "md5sum": "3a8f050e1c81d9c21bd78dc7df0d4bc4" + "md5sum": "69af3fefd0ad33870219cab220d98c1b" }, { - "dataPath": "params_shard_26.bin", + "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.19.mlp.down_proj.weight", + "name": "model.layers.10.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -981,15 +932,15 @@ "byteOffset": 0 } ], - "md5sum": "36ca71a16ca2834d3164b92c97a170ab" + "md5sum": "5862aaa3d0c9514ced28f0c391b70d8c" }, { - "dataPath": "params_shard_27.bin", + "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.19.mlp.gate_up_proj.weight", + "name": "model.layers.11.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1000,15 +951,15 @@ "byteOffset": 0 } ], - "md5sum": "9f22ca095f2021608fae9bc98136d252" + "md5sum": "221ad0e352c9a06fbd95fa8990829aac" }, { - "dataPath": "params_shard_28.bin", + "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.2.mlp.down_proj.weight", + "name": "model.layers.11.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1019,34 +970,15 @@ "byteOffset": 0 } ], - "md5sum": "fab4a0e7de9cca7058251ee57a4a9742" - }, - { - "dataPath": "params_shard_29.bin", - "format": "raw-shard", - "nbytes": 55050240, - "records": [ - { - "name": "model.layers.2.mlp.gate_up_proj.weight", - "shape": [ - 17920, - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 55050240, - "byteOffset": 0 - } - ], - "md5sum": "e6b441ea46b5d4ba144d0ede689da1a2" + "md5sum": "605ca260375f758e22d4265fffa356fc" }, { - "dataPath": "params_shard_30.bin", + "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.17.self_attn.c_attn.weight", + "name": "model.layers.9.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1057,48 +989,48 @@ "byteOffset": 0 }, { - "name": "model.layers.17.self_attn.o_proj.weight", + "name": "model.layers.9.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.18.input_layernorm.weight", + "name": "model.layers.9.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.18.post_attention_layernorm.weight", + "name": "model.layers.9.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.18.self_attn.c_attn.bias", + "name": "model.layers.9.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.18.self_attn.c_attn.weight", + "name": "model.layers.10.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1109,48 +1041,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.18.self_attn.o_proj.weight", + "name": "model.layers.10.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.19.input_layernorm.weight", + "name": "model.layers.10.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.19.post_attention_layernorm.weight", + "name": "model.layers.10.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.19.self_attn.c_attn.bias", + "name": "model.layers.10.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.19.self_attn.c_attn.weight", + "name": "model.layers.11.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1161,75 +1093,56 @@ "byteOffset": 22040576 }, { - "name": "model.layers.19.self_attn.o_proj.weight", + "name": "model.layers.11.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 28332032 }, { - "name": "model.layers.2.input_layernorm.weight", + "name": "model.layers.11.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 33050624 + "nbytes": 4718592, + "byteOffset": 28336128 }, { - "name": "model.layers.2.post_attention_layernorm.weight", + "name": "model.layers.11.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 + "byteOffset": 33054720 }, { - "name": "model.layers.2.self_attn.c_attn.bias", + "name": "model.layers.11.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 3072, + "byteOffset": 33057792 } ], - "md5sum": "ec32c77b7b448670b9fbb3e4dadd2b9f" + "md5sum": "03e434a9483fb2aaa19f4cab3e4fb083" }, { - "dataPath": "params_shard_31.bin", + "dataPath": "params_shard_29.bin", "format": "raw-shard", - "nbytes": 27525120, + "nbytes": 55050240, "records": [ { - "name": "model.layers.20.mlp.down_proj.weight", - "shape": [ - 1536, - 8960 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 27525120, - "byteOffset": 0 - } - ], - "md5sum": "169d2af7bb908d1d1319b2327ffd685d" - }, - { - "dataPath": "params_shard_32.bin", - "format": "raw-shard", - "nbytes": 55050240, - "records": [ - { - "name": "model.layers.20.mlp.gate_up_proj.weight", + "name": "model.layers.12.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1240,15 +1153,15 @@ "byteOffset": 0 } ], - "md5sum": "549d4ada0599a1d35bf7dd3f87dfad84" + "md5sum": "dc39bbf30e260c4170b068cc8553d7cd" }, { - "dataPath": "params_shard_33.bin", + "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.21.mlp.down_proj.weight", + "name": "model.layers.12.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1259,15 +1172,15 @@ "byteOffset": 0 } ], - "md5sum": "6ea484043810c0eeff8409f4fffc7c14" + "md5sum": "d6ebf5371ced23357eb54793db3a4afb" }, { - "dataPath": "params_shard_34.bin", + "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.21.mlp.gate_up_proj.weight", + "name": "model.layers.13.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1278,15 +1191,15 @@ "byteOffset": 0 } ], - "md5sum": "f09443ff1e071a170ee446e8b5c9e983" + "md5sum": "b23a6222ce16baad47b61fe41f8660b0" }, { - "dataPath": "params_shard_35.bin", + "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.22.mlp.down_proj.weight", + "name": "model.layers.13.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1297,15 +1210,15 @@ "byteOffset": 0 } ], - "md5sum": "367fc7511fd53ad92f83eff81ee49dc2" + "md5sum": "f579126381720332c477dfeb50701e20" }, { - "dataPath": "params_shard_36.bin", + "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.22.mlp.gate_up_proj.weight", + "name": "model.layers.14.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1316,15 +1229,34 @@ "byteOffset": 0 } ], - "md5sum": "0a8e85d059205dd0b12cbfd488dbcc1f" + "md5sum": "44b78e073ac8337e052b9744380ec3c7" }, { - "dataPath": "params_shard_37.bin", + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 27525120, + "records": [ + { + "name": "model.layers.14.mlp.down_proj.weight", + "shape": [ + 1536, + 8960 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 27525120, + "byteOffset": 0 + } + ], + "md5sum": "d205aa713ee99dee09b571cfedc7cefd" + }, + { + "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.2.self_attn.c_attn.weight", + "name": "model.layers.12.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1335,48 +1267,48 @@ "byteOffset": 0 }, { - "name": "model.layers.2.self_attn.o_proj.weight", + "name": "model.layers.12.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.20.input_layernorm.weight", + "name": "model.layers.12.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.20.post_attention_layernorm.weight", + "name": "model.layers.12.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.20.self_attn.c_attn.bias", + "name": "model.layers.12.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.20.self_attn.c_attn.weight", + "name": "model.layers.13.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1387,48 +1319,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.20.self_attn.o_proj.weight", + "name": "model.layers.13.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.21.input_layernorm.weight", + "name": "model.layers.13.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.21.post_attention_layernorm.weight", + "name": "model.layers.13.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.21.self_attn.c_attn.bias", + "name": "model.layers.13.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.21.self_attn.c_attn.weight", + "name": "model.layers.14.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1439,7 +1371,17 @@ "byteOffset": 22040576 }, { - "name": "model.layers.21.self_attn.o_proj.weight", + "name": "model.layers.14.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 28332032 + }, + { + "name": "model.layers.14.self_attn.o_proj.weight", "shape": [ 1536, 1536 @@ -1447,48 +1389,57 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, - "byteOffset": 28332032 + "byteOffset": 28336128 }, { - "name": "model.layers.22.input_layernorm.weight", + "name": "model.layers.14.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33050624 + "byteOffset": 33054720 }, { - "name": "model.layers.22.post_attention_layernorm.weight", + "name": "model.layers.14.post_attention_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 - }, + "byteOffset": 33057792 + } + ], + "md5sum": "54c052279cd7ffad4678b87826192f76" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 55050240, + "records": [ { - "name": "model.layers.22.self_attn.c_attn.bias", + "name": "model.layers.15.mlp.gate_up_proj.weight", "shape": [ - 2048 + 17920, + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 55050240, + "byteOffset": 0 } ], - "md5sum": "670d9f64575bb07241d48a8283edc7c3" + "md5sum": "0dac5ce31c1f2eaa17a56d0da55f7b6d" }, { - "dataPath": "params_shard_38.bin", + "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.23.mlp.down_proj.weight", + "name": "model.layers.15.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1499,15 +1450,15 @@ "byteOffset": 0 } ], - "md5sum": "55dc4417efdd8592cf2596108dc49e03" + "md5sum": "6b9f7819701341646bc2a266108f95b2" }, { - "dataPath": "params_shard_39.bin", + "dataPath": "params_shard_38.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.23.mlp.gate_up_proj.weight", + "name": "model.layers.16.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1518,15 +1469,15 @@ "byteOffset": 0 } ], - "md5sum": "59aae069c3cf2e5dfc148b92384dea3d" + "md5sum": "04cd76c5b6fc4ebe6baaa8644b6930fd" }, { - "dataPath": "params_shard_40.bin", + "dataPath": "params_shard_39.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.24.mlp.down_proj.weight", + "name": "model.layers.16.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1537,15 +1488,15 @@ "byteOffset": 0 } ], - "md5sum": "3ee6d1d9d6a0d40aa5c6cda617a5d1bb" + "md5sum": "349f0c70d75d8a410f1b236fd7e395b5" }, { - "dataPath": "params_shard_41.bin", + "dataPath": "params_shard_40.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.24.mlp.gate_up_proj.weight", + "name": "model.layers.17.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1556,15 +1507,15 @@ "byteOffset": 0 } ], - "md5sum": "a1c9e0afa3d19157d9470ecbe8d02441" + "md5sum": "63960103006f54f63a378784156e1ff6" }, { - "dataPath": "params_shard_42.bin", + "dataPath": "params_shard_41.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.25.mlp.down_proj.weight", + "name": "model.layers.17.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1575,34 +1526,15 @@ "byteOffset": 0 } ], - "md5sum": "0c88c6fbbe93a15c6d1c34b64769e9ee" - }, - { - "dataPath": "params_shard_43.bin", - "format": "raw-shard", - "nbytes": 55050240, - "records": [ - { - "name": "model.layers.25.mlp.gate_up_proj.weight", - "shape": [ - 17920, - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 55050240, - "byteOffset": 0 - } - ], - "md5sum": "bf742044703c2a29f0c83f872a30751a" + "md5sum": "0f07069c62ca1e2d8c3e7ee712fe82b0" }, { - "dataPath": "params_shard_44.bin", + "dataPath": "params_shard_42.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.22.self_attn.c_attn.weight", + "name": "model.layers.15.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1613,48 +1545,48 @@ "byteOffset": 0 }, { - "name": "model.layers.22.self_attn.o_proj.weight", + "name": "model.layers.15.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.23.input_layernorm.weight", + "name": "model.layers.15.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.23.post_attention_layernorm.weight", + "name": "model.layers.15.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.23.self_attn.c_attn.bias", + "name": "model.layers.15.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.23.self_attn.c_attn.weight", + "name": "model.layers.16.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1665,48 +1597,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.23.self_attn.o_proj.weight", + "name": "model.layers.16.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.24.input_layernorm.weight", + "name": "model.layers.16.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.24.post_attention_layernorm.weight", + "name": "model.layers.16.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.24.self_attn.c_attn.bias", + "name": "model.layers.16.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.24.self_attn.c_attn.weight", + "name": "model.layers.17.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1717,94 +1649,75 @@ "byteOffset": 22040576 }, { - "name": "model.layers.24.self_attn.o_proj.weight", + "name": "model.layers.17.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 28332032 }, { - "name": "model.layers.25.input_layernorm.weight", + "name": "model.layers.17.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 33050624 + "nbytes": 4718592, + "byteOffset": 28336128 }, { - "name": "model.layers.25.post_attention_layernorm.weight", + "name": "model.layers.17.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 + "byteOffset": 33054720 }, { - "name": "model.layers.25.self_attn.c_attn.bias", + "name": "model.layers.17.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 3072, + "byteOffset": 33057792 } ], - "md5sum": "9bd20d328ef42e9cb0daeea34f8f1665" + "md5sum": "cbdd25fd19637b648dff1256b0362d00" }, { - "dataPath": "params_shard_45.bin", + "dataPath": "params_shard_43.bin", "format": "raw-shard", - "nbytes": 27525120, + "nbytes": 55050240, "records": [ { - "name": "model.layers.26.mlp.down_proj.weight", + "name": "model.layers.18.mlp.gate_up_proj.weight", "shape": [ - 1536, - 8960 + 17920, + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 27525120, + "nbytes": 55050240, "byteOffset": 0 } ], - "md5sum": "3bf660c4dd495b4cb010e090958b399f" + "md5sum": "639c98457bec8c7e83c2c1da8b9410f2" }, { - "dataPath": "params_shard_46.bin", + "dataPath": "params_shard_44.bin", "format": "raw-shard", - "nbytes": 55050240, + "nbytes": 27525120, "records": [ { - "name": "model.layers.26.mlp.gate_up_proj.weight", - "shape": [ - 17920, - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 55050240, - "byteOffset": 0 - } - ], - "md5sum": "1d8d43c1f77c22de3ed30d08f84db8c1" - }, - { - "dataPath": "params_shard_47.bin", - "format": "raw-shard", - "nbytes": 27525120, - "records": [ - { - "name": "model.layers.27.mlp.down_proj.weight", + "name": "model.layers.18.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1815,15 +1728,15 @@ "byteOffset": 0 } ], - "md5sum": "5dba30ad09b4e0832d4b66cbab7cbb65" + "md5sum": "c335e79b08a2fe03d63fa514b658d909" }, { - "dataPath": "params_shard_48.bin", + "dataPath": "params_shard_45.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.27.mlp.gate_up_proj.weight", + "name": "model.layers.19.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1834,15 +1747,15 @@ "byteOffset": 0 } ], - "md5sum": "3e3f1c5f902e1612e29d8d4bfce8c13f" + "md5sum": "6e0525006bc32f3ec81492080dc9f7f2" }, { - "dataPath": "params_shard_49.bin", + "dataPath": "params_shard_46.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.3.mlp.down_proj.weight", + "name": "model.layers.19.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -1853,15 +1766,15 @@ "byteOffset": 0 } ], - "md5sum": "dafb2ba9ed30881ead0f758444c710e9" + "md5sum": "d968c5af427bccf73fb824e0be561cd9" }, { - "dataPath": "params_shard_50.bin", + "dataPath": "params_shard_47.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.3.mlp.gate_up_proj.weight", + "name": "model.layers.20.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -1872,15 +1785,34 @@ "byteOffset": 0 } ], - "md5sum": "fae672c0483259b7c6f78ebe8c9bc007" + "md5sum": "80e8b32cb11f59a6022acf852eea7ace" }, { - "dataPath": "params_shard_51.bin", + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 27525120, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.weight", + "shape": [ + 1536, + 8960 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 27525120, + "byteOffset": 0 + } + ], + "md5sum": "c075bcffc12a1d712ca76c1471ae444a" + }, + { + "dataPath": "params_shard_49.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.25.self_attn.c_attn.weight", + "name": "model.layers.18.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1891,48 +1823,48 @@ "byteOffset": 0 }, { - "name": "model.layers.25.self_attn.o_proj.weight", + "name": "model.layers.18.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.26.input_layernorm.weight", + "name": "model.layers.18.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.26.post_attention_layernorm.weight", + "name": "model.layers.18.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.26.self_attn.c_attn.bias", + "name": "model.layers.18.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.26.self_attn.c_attn.weight", + "name": "model.layers.19.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1943,48 +1875,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.26.self_attn.o_proj.weight", + "name": "model.layers.19.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.27.input_layernorm.weight", + "name": "model.layers.19.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.27.post_attention_layernorm.weight", + "name": "model.layers.19.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.27.self_attn.c_attn.bias", + "name": "model.layers.19.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.27.self_attn.c_attn.weight", + "name": "model.layers.20.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -1995,7 +1927,17 @@ "byteOffset": 22040576 }, { - "name": "model.layers.27.self_attn.o_proj.weight", + "name": "model.layers.20.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 28332032 + }, + { + "name": "model.layers.20.self_attn.o_proj.weight", "shape": [ 1536, 1536 @@ -2003,48 +1945,57 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, - "byteOffset": 28332032 + "byteOffset": 28336128 }, { - "name": "model.layers.3.input_layernorm.weight", + "name": "model.layers.20.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33050624 + "byteOffset": 33054720 }, { - "name": "model.layers.3.post_attention_layernorm.weight", + "name": "model.layers.20.post_attention_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 - }, + "byteOffset": 33057792 + } + ], + "md5sum": "73cd2787ae936b6aa7f0c3c9c85b3ff2" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 55050240, + "records": [ { - "name": "model.layers.3.self_attn.c_attn.bias", + "name": "model.layers.21.mlp.gate_up_proj.weight", "shape": [ - 2048 + 17920, + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 55050240, + "byteOffset": 0 } ], - "md5sum": "e15198cbf013718fbac52a1a8d94a4bd" + "md5sum": "136a14789a41480fcc08d89e66ca2501" }, { - "dataPath": "params_shard_52.bin", + "dataPath": "params_shard_51.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.4.mlp.down_proj.weight", + "name": "model.layers.21.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -2055,15 +2006,15 @@ "byteOffset": 0 } ], - "md5sum": "8ef490e2d5fdd10b563790c6c8ddf751" + "md5sum": "a3f54fbb03f00cc32caaa8b0a5cb8394" }, { - "dataPath": "params_shard_53.bin", + "dataPath": "params_shard_52.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.4.mlp.gate_up_proj.weight", + "name": "model.layers.22.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -2074,15 +2025,15 @@ "byteOffset": 0 } ], - "md5sum": "0d9f22936600b212ad7509684535ed16" + "md5sum": "2ffd1ebbe30c22f30f61a3d92c032d1b" }, { - "dataPath": "params_shard_54.bin", + "dataPath": "params_shard_53.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.5.mlp.down_proj.weight", + "name": "model.layers.22.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -2093,15 +2044,15 @@ "byteOffset": 0 } ], - "md5sum": "b877bb9f1825407f4828cde2906e483c" + "md5sum": "2a50d409322f1488c49d5287dd21c3b8" }, { - "dataPath": "params_shard_55.bin", + "dataPath": "params_shard_54.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.5.mlp.gate_up_proj.weight", + "name": "model.layers.23.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -2112,15 +2063,15 @@ "byteOffset": 0 } ], - "md5sum": "4aeb2df9e320197a7328f274ccc1e0c6" + "md5sum": "89a049c13e34f896861d5989e7af37ca" }, { - "dataPath": "params_shard_56.bin", + "dataPath": "params_shard_55.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.6.mlp.down_proj.weight", + "name": "model.layers.23.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -2131,34 +2082,15 @@ "byteOffset": 0 } ], - "md5sum": "4159b8469797370880dec9cef854c8b1" - }, - { - "dataPath": "params_shard_57.bin", - "format": "raw-shard", - "nbytes": 55050240, - "records": [ - { - "name": "model.layers.6.mlp.gate_up_proj.weight", - "shape": [ - 17920, - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 55050240, - "byteOffset": 0 - } - ], - "md5sum": "f6f34569657363a613bb38ffa4a8ce25" + "md5sum": "60e61d289310f677b122617d45a9c7e4" }, { - "dataPath": "params_shard_58.bin", + "dataPath": "params_shard_56.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.3.self_attn.c_attn.weight", + "name": "model.layers.21.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -2169,48 +2101,48 @@ "byteOffset": 0 }, { - "name": "model.layers.3.self_attn.o_proj.weight", + "name": "model.layers.21.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.4.input_layernorm.weight", + "name": "model.layers.21.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.4.post_attention_layernorm.weight", + "name": "model.layers.21.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.4.self_attn.c_attn.bias", + "name": "model.layers.21.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.4.self_attn.c_attn.weight", + "name": "model.layers.22.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -2221,48 +2153,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.4.self_attn.o_proj.weight", + "name": "model.layers.22.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.5.input_layernorm.weight", + "name": "model.layers.22.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.5.post_attention_layernorm.weight", + "name": "model.layers.22.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.5.self_attn.c_attn.bias", + "name": "model.layers.22.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.5.self_attn.c_attn.weight", + "name": "model.layers.23.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -2273,7 +2205,17 @@ "byteOffset": 22040576 }, { - "name": "model.layers.5.self_attn.o_proj.weight", + "name": "model.layers.23.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 28332032 + }, + { + "name": "model.layers.23.self_attn.o_proj.weight", "shape": [ 1536, 1536 @@ -2281,48 +2223,57 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, - "byteOffset": 28332032 + "byteOffset": 28336128 }, { - "name": "model.layers.6.input_layernorm.weight", + "name": "model.layers.23.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33050624 + "byteOffset": 33054720 }, { - "name": "model.layers.6.post_attention_layernorm.weight", + "name": "model.layers.23.post_attention_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 - }, + "byteOffset": 33057792 + } + ], + "md5sum": "2859dd48c6a504a6ff5fc777a5d0eb01" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 55050240, + "records": [ { - "name": "model.layers.6.self_attn.c_attn.bias", + "name": "model.layers.24.mlp.gate_up_proj.weight", "shape": [ - 2048 + 17920, + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 55050240, + "byteOffset": 0 } ], - "md5sum": "4b9635bfbb9a0f8e379594d577e24e79" + "md5sum": "00c023336d6889bb9d24bcb3762ff819" }, { - "dataPath": "params_shard_59.bin", + "dataPath": "params_shard_58.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.7.mlp.down_proj.weight", + "name": "model.layers.24.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -2333,15 +2284,15 @@ "byteOffset": 0 } ], - "md5sum": "6f06f08ee4e830a7f35fad5489e2df96" + "md5sum": "725c3d334464020a85016e629f58eda8" }, { - "dataPath": "params_shard_60.bin", + "dataPath": "params_shard_59.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.7.mlp.gate_up_proj.weight", + "name": "model.layers.25.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -2352,15 +2303,15 @@ "byteOffset": 0 } ], - "md5sum": "6a51b2acc4eb22511d78c73146e97c57" + "md5sum": "09c4056fb69cfcce7f2c9829ba155e28" }, { - "dataPath": "params_shard_61.bin", + "dataPath": "params_shard_60.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.8.mlp.down_proj.weight", + "name": "model.layers.25.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -2371,15 +2322,15 @@ "byteOffset": 0 } ], - "md5sum": "422c9f65efd3c517f5f597ff198b9b54" + "md5sum": "898fa6e97b95ddc00a3a685714dd2056" }, { - "dataPath": "params_shard_62.bin", + "dataPath": "params_shard_61.bin", "format": "raw-shard", "nbytes": 55050240, "records": [ { - "name": "model.layers.8.mlp.gate_up_proj.weight", + "name": "model.layers.26.mlp.gate_up_proj.weight", "shape": [ 17920, 1536 @@ -2390,15 +2341,15 @@ "byteOffset": 0 } ], - "md5sum": "194381860a8101e6639b37c321b03e53" + "md5sum": "ae5ce4d265237aae0fd576bb6f528dca" }, { - "dataPath": "params_shard_63.bin", + "dataPath": "params_shard_62.bin", "format": "raw-shard", "nbytes": 27525120, "records": [ { - "name": "model.layers.9.mlp.down_proj.weight", + "name": "model.layers.26.mlp.down_proj.weight", "shape": [ 1536, 8960 @@ -2409,34 +2360,15 @@ "byteOffset": 0 } ], - "md5sum": "dcc2852e43b9cca8d247abb596a6cfd3" + "md5sum": "58375129ef74f23538d522ef09315b51" }, { - "dataPath": "params_shard_64.bin", - "format": "raw-shard", - "nbytes": 55050240, - "records": [ - { - "name": "model.layers.9.mlp.gate_up_proj.weight", - "shape": [ - 17920, - 1536 - ], - "dtype": "float16", - "format": "f32-to-bf16", - "nbytes": 55050240, - "byteOffset": 0 - } - ], - "md5sum": "31b0adf54bda832e421327cc270f38de" - }, - { - "dataPath": "params_shard_65.bin", + "dataPath": "params_shard_63.bin", "format": "raw-shard", "nbytes": 33060864, "records": [ { - "name": "model.layers.6.self_attn.c_attn.weight", + "name": "model.layers.24.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -2447,48 +2379,48 @@ "byteOffset": 0 }, { - "name": "model.layers.6.self_attn.o_proj.weight", + "name": "model.layers.24.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 6291456 }, { - "name": "model.layers.7.input_layernorm.weight", + "name": "model.layers.24.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 11010048 + "nbytes": 4718592, + "byteOffset": 6295552 }, { - "name": "model.layers.7.post_attention_layernorm.weight", + "name": "model.layers.24.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11013120 + "byteOffset": 11014144 }, { - "name": "model.layers.7.self_attn.c_attn.bias", + "name": "model.layers.24.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 11016192 + "nbytes": 3072, + "byteOffset": 11017216 }, { - "name": "model.layers.7.self_attn.c_attn.weight", + "name": "model.layers.25.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -2499,48 +2431,48 @@ "byteOffset": 11020288 }, { - "name": "model.layers.7.self_attn.o_proj.weight", + "name": "model.layers.25.self_attn.c_attn.bias", "shape": [ - 1536, - 1536 + 2048 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4718592, + "nbytes": 4096, "byteOffset": 17311744 }, { - "name": "model.layers.8.input_layernorm.weight", + "name": "model.layers.25.self_attn.o_proj.weight", "shape": [ + 1536, 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 3072, - "byteOffset": 22030336 + "nbytes": 4718592, + "byteOffset": 17315840 }, { - "name": "model.layers.8.post_attention_layernorm.weight", + "name": "model.layers.25.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 22033408 + "byteOffset": 22034432 }, { - "name": "model.layers.8.self_attn.c_attn.bias", + "name": "model.layers.25.post_attention_layernorm.weight", "shape": [ - 2048 + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 22036480 + "nbytes": 3072, + "byteOffset": 22037504 }, { - "name": "model.layers.8.self_attn.c_attn.weight", + "name": "model.layers.26.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -2551,7 +2483,17 @@ "byteOffset": 22040576 }, { - "name": "model.layers.8.self_attn.o_proj.weight", + "name": "model.layers.26.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 28332032 + }, + { + "name": "model.layers.26.self_attn.o_proj.weight", "shape": [ 1536, 1536 @@ -2559,48 +2501,76 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, - "byteOffset": 28332032 + "byteOffset": 28336128 }, { - "name": "model.layers.9.input_layernorm.weight", + "name": "model.layers.26.input_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33050624 + "byteOffset": 33054720 }, { - "name": "model.layers.9.post_attention_layernorm.weight", + "name": "model.layers.26.post_attention_layernorm.weight", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 33053696 - }, + "byteOffset": 33057792 + } + ], + "md5sum": "50113496b65c91d08a408ca51382050c" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 55050240, + "records": [ { - "name": "model.layers.9.self_attn.c_attn.bias", + "name": "model.layers.27.mlp.gate_up_proj.weight", "shape": [ - 2048 + 17920, + 1536 ], "dtype": "float16", "format": "f32-to-bf16", - "nbytes": 4096, - "byteOffset": 33056768 + "nbytes": 55050240, + "byteOffset": 0 } ], - "md5sum": "80e3191e706ecd10535e81ffa38fc3c2" + "md5sum": "77bba71a9b67ed4377d409949e9ad145" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 27525120, + "records": [ + { + "name": "model.layers.27.mlp.down_proj.weight", + "shape": [ + 1536, + 8960 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 27525120, + "byteOffset": 0 + } + ], + "md5sum": "0d6ad7eb791649a022f869b1edeb5355" }, { "dataPath": "params_shard_66.bin", "format": "raw-shard", - "nbytes": 11013120, + "nbytes": 11023360, "records": [ { - "name": "model.layers.9.self_attn.c_attn.weight", + "name": "model.layers.27.self_attn.c_attn.weight", "shape": [ 2048, 1536 @@ -2611,7 +2581,17 @@ "byteOffset": 0 }, { - "name": "model.layers.9.self_attn.o_proj.weight", + "name": "model.layers.27.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 6291456 + }, + { + "name": "model.layers.27.self_attn.o_proj.weight", "shape": [ 1536, 1536 @@ -2619,7 +2599,27 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, - "byteOffset": 6291456 + "byteOffset": 6295552 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 11014144 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 11017216 }, { "name": "model.norm.weight", @@ -2629,10 +2629,10 @@ "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, - "byteOffset": 11010048 + "byteOffset": 11020288 } ], - "md5sum": "554e351ae89de4c09af73e23fd71576f" + "md5sum": "29c619ad6174a67fe28726e26cc3974d" } ] } \ No newline at end of file