Upload MistralForCausalLM

config.json

@@ -1,27 +1,25 @@
 {
-  "_name_or_path": "codellama/CodeLlama-7b-hf",
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
   "architectures": [
-    "LlamaForCausalLM"
+    "MistralForCausalLM"
   ],
-  "attention_bias": false,
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
-  "intermediate_size": 11008,
-  "max_position_embeddings": 16384,
-  "model_type": "llama",
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "pretraining_tp": 1,
+  "num_key_value_heads": 8,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 1000000,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "float16",
   "transformers_version": "4.35.2",
   "use_cache": true,
-  "vocab_size": 32016
+  "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:430a5f0f26f51c534feb90ee77c7d09aca503a832b4d8489c6dda7d3aacc10ad
-size 4939116320
+oid sha256:77c79484ba363b5f28fa02884be1c0851bd55ef7bf8a7bf59b3ca595eda11be6
+size 4943162240

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87329c01529394b334f8eabb5b4cf344359c2ed60ba07bd36788728f7bf09a91
-size 4947390768
+oid sha256:fe6e95bd8594296699686dcca0da98f0434321ef01d2b0b3911ad793b9741164
+size 4999819232

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9a1d9f237267298a517d053f6d15cf2620e3d10ea2e7003492fee8e31b1db14
-size 3590619808
+oid sha256:7b453ec6227016e3989557a1927d152de8434c69ffac13cd04adb11a7bfa07dc
+size 4540516256

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 13477093376
+    "total_size": 14483464192
   },
   "weight_map": {
     "lm_head.weight": "model-00003-of-00003.safetensors",
@@ -23,24 +23,24 @@
     "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
@@ -140,24 +140,24 @@
     "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",