Update config.json

config.json

@@ -1,38 +1,29 @@
-{"act_fn": None,
- "attention_dir": "causal",
- "attn_only": True,
- "attn_types": None,
- "checkpoint_index": None,
- "checkpoint_label_type": None,
- "checkpoint_value": None,
- "d_head": 32,
- "d_mlp": None,
- "d_model": 256,
- "d_vocab": 50259,
- "d_vocab_out": 50259,
- "device": "cuda",
- "eps": 1e-05,
- "final_rms": False,
- "from_checkpoint": False,
- "init_mode": "gpt2",
- "init_weights": True,
- "initializer_range": 0.05,
- "model_name": "custom",
- "n_ctx": 2048,
- "n_heads": 8,
- "n_layers": 2,
- "n_params": 524288,
- "normalization_type": "LN",
- "original_architecture": None,
- "parallel_attn_mlp": False,
- "positional_embedding_type": "shortformer",
- "rotary_dim": None,
- "scale_attn_by_inverse_layer_idx": False,
- "seed": None,
- "tokenizer_name": None,
- "use_attn_result": True,
- "use_attn_scale": True,
- "use_hook_tokens": False,
- "use_local_attn": False,
- "use_split_qkv_input": False,
- "window_size": None}
+{
+  "n_layers": 2,
+  "d_model": 256,
+  "n_ctx": 2048,
+  "d_head": 32,
+  "model_name": "custom",
+  "n_heads": 8,
+  "d_vocab": 50259,
+  "eps": 1e-05,
+  "use_attn_result": True,
+  "use_attn_scale": True,
+  "use_split_qkv_input": False,
+  "use_local_attn": False,
+  "from_checkpoint": False,
+  "init_mode": "gpt2",
+  "normalization_type": "LN",
+  "device": "cuda",
+  "attention_dir": "causal",
+  "attn_only": True,
+  "initializer_range": 0.05,
+  "init_weights": True,
+  "scale_attn_by_inverse_layer_idx": False,
+  "positional_embedding_type": "shortformer",
+  "final_rms": False,
+  "d_vocab_out": 50259,
+  "parallel_attn_mlp": False,
+  "n_params": 524288,
+  "use_hook_tokens": False,
+}