Upload LlamaForCausalLM

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "Vikhrmodels/Vikhr-7b-0.2",
+  "_name_or_path": "AlexWortega/v4",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -10,20 +10,20 @@
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
-  "intermediate_size": 11008,
-  "max_position_embeddings": 2048,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "pad_token_id": 0,
+  "num_key_value_heads": 8,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 10000.0,
+  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.1",
-  "use_cache": false,
-  "vocab_size": 60258
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 40002
 }

generation_config.json

@@ -2,7 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "pad_token_id": 0,
-  "transformers_version": "4.39.1",
-  "use_cache": false
+  "transformers_version": "4.39.3"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88aa6434480d8cdd983dc47ae2520b527f51486dc6c9c33acd1ff7c1e38420a5
-size 4979633592
+oid sha256:1b1b4aaba96519930d4d94186ba3d11855149ca04779df9c74e63aa0cd362d1c
+size 4891274096

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2a0aee196878f8a241d4aa98dd82abe72934c87423a73dc204cb608b23218f9
-size 4957876888
+oid sha256:41cf28dc730105259f0f14bb7f89e2adde6d2e6aabfe2968a5e64708586a1db5
+size 4915916176

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e038ddce280fccdad76371f29f847b2670948302b4a61f723c737a14a6df771
-size 4002333672
+oid sha256:7b7f220d3c530e549aa7be37d651aa1c8c9147b7f7ba0ee1fc8b02b69c91b439
+size 4807412520

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 13939810304
+    "total_size": 14614568960
   },
   "weight_map": {
     "lm_head.weight": "model-00003-of-00003.safetensors",
@@ -23,11 +23,11 @@
     "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
@@ -39,7 +39,7 @@
     "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
@@ -131,33 +131,33 @@
     "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",