{
  "n_layers": 2,
  "d_model": 256,
  "n_ctx": 2048,
  "d_head": 32,
  "model_name": "custom",
  "n_heads": 8,
  "d_vocab": 50259,
  "eps": 1e-05,
  "use_attn_result": True,
  "use_attn_scale": True,
  "use_split_qkv_input": False,
  "use_local_attn": False,
  "from_checkpoint": False,
  "init_mode": "gpt2",
  "normalization_type": "LN",
  "device": "cuda",
  "attention_dir": "causal",
  "attn_only": True,
  "initializer_range": 0.05,
  "init_weights": True,
  "scale_attn_by_inverse_layer_idx": False,
  "positional_embedding_type": "shortformer",
  "final_rms": False,
  "d_vocab_out": 50259,
  "parallel_attn_mlp": False,
  "n_params": 524288,
  "use_hook_tokens": False,
}