ArthurConmy
commited on
Commit
•
e8173c9
1
Parent(s):
badf588
Fix config
Browse files- config.json +11 -11
config.json
CHANGED
@@ -7,24 +7,24 @@
|
|
7 |
"n_heads": 8,
|
8 |
"d_vocab": 50259,
|
9 |
"eps": 1e-05,
|
10 |
-
"use_attn_result":
|
11 |
-
"use_attn_scale":
|
12 |
-
"use_split_qkv_input":
|
13 |
-
"use_local_attn":
|
14 |
-
"from_checkpoint":
|
15 |
"init_mode": "gpt2",
|
16 |
"normalization_type": "LN",
|
17 |
"device": "cuda",
|
18 |
"attention_dir": "causal",
|
19 |
-
"attn_only":
|
20 |
"initializer_range": 0.05,
|
21 |
-
"init_weights":
|
22 |
-
"scale_attn_by_inverse_layer_idx":
|
23 |
"positional_embedding_type": "shortformer",
|
24 |
-
"final_rms":
|
25 |
"d_vocab_out": 50259,
|
26 |
-
"parallel_attn_mlp":
|
27 |
"n_params": 524288,
|
28 |
-
"use_hook_tokens":
|
29 |
"tokenizer_name": "ArthurConmy/redwood_tokenizer"
|
30 |
}
|
|
|
7 |
"n_heads": 8,
|
8 |
"d_vocab": 50259,
|
9 |
"eps": 1e-05,
|
10 |
+
"use_attn_result": true,
|
11 |
+
"use_attn_scale": true,
|
12 |
+
"use_split_qkv_input": false,
|
13 |
+
"use_local_attn": false,
|
14 |
+
"from_checkpoint": false,
|
15 |
"init_mode": "gpt2",
|
16 |
"normalization_type": "LN",
|
17 |
"device": "cuda",
|
18 |
"attention_dir": "causal",
|
19 |
+
"attn_only": true,
|
20 |
"initializer_range": 0.05,
|
21 |
+
"init_weights": true,
|
22 |
+
"scale_attn_by_inverse_layer_idx": false,
|
23 |
"positional_embedding_type": "shortformer",
|
24 |
+
"final_rms": false,
|
25 |
"d_vocab_out": 50259,
|
26 |
+
"parallel_attn_mlp": false,
|
27 |
"n_params": 524288,
|
28 |
+
"use_hook_tokens": false,
|
29 |
"tokenizer_name": "ArthurConmy/redwood_tokenizer"
|
30 |
}
|