Update config.json

config.json

@@ -13,6 +13,7 @@
   "initializer_range": 0.02,
   "intermediate_size": 14336,
   "max_position_embeddings": 8192,
+  "max_sequence_length": 32768,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_experts": 2,
@@ -21,55 +22,15 @@
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
-    "rope_scaling": {
+  "rope_scaling": {
     "factor": 4.0,
     "original_max_position_embeddings": 8192,
     "type": "yarn",
     "finetuned": "true"
   },
-  "rope_theta": 16000000.0,
-  "router_layers": [
-    "gate_proj",
-    "up_proj",
-    "down_proj"
-  ],
-  "router_layers_index": [
-    0,
-    1,
-    2,
-    3,
-    4,
-    5,
-    6,
-    7,
-    8,
-    9,
-    10,
-    11,
-    12,
-    13,
-    14,
-    15,
-    16,
-    17,
-    18,
-    19,
-    20,
-    21,
-    22,
-    23,
-    24,
-    25,
-    26,
-    27,
-    28,
-    29,
-    30,
-    31
-  ],
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.1",
+  "transformers_version": "4.40.0.dev0",
   "use_cache": true,
   "vocab_size": 128256
 }