joey00072
/

expriment-mla

model_hub_mixin

pytorch_model_hub_mixin

Model card Files Files and versions Community

joey00072 commited on Oct 17

Commit

8aa5e3b

•

1 Parent(s): 0143254

checkpoint iter: 2000

Files changed (2) hide show

config.json +4 -4
model.safetensors +2 -2

config.json CHANGED Viewed

@@ -6,12 +6,12 @@
   "dropout": 0.2,
   "head_dim": 64,
   "hidden_dim": 1536,
-  "kv_lora_rank": 64,
   "mlp": "GLU",
-  "num_heads": 46,
-  "num_kv_heads": 46,
   "num_layers": 4,
-  "q_lora_rank": 192,
   "rope_head_dim": 16,
   "seq_len": 256,
   "vocab_size": 50257,

   "dropout": 0.2,
   "head_dim": 64,
   "hidden_dim": 1536,
+  "kv_lora_rank": 128,
   "mlp": "GLU",
+  "num_heads": 32,
+  "num_kv_heads": 32,
   "num_layers": 4,
+  "q_lora_rank": 384,
   "rope_head_dim": 16,
   "seq_len": 256,
   "vocab_size": 50257,

model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f0b81b4e8bd1a01b4825d2d3dfe324ad7018a19c9aed95ad49aa0c5342ec2a2
-size 554294048

 version https://git-lfs.github.com/spec/v1
+oid sha256:cd29eac3a78e43c10426aef42f84e4babfc61950a18d4c255e0bf2202e329b75
+size 549415728