{
  "embed_dim": 768,
  "num_heads": 4,
  "dropout": 0.025,
  "bias": true,
  "gate_min": 0.05,
  "gate_max": 0.95,
  "gate_dropout": 0.05,
  "dropout_gate_open": 0.0,
  "dropout_gate_close": 0.0,
  "CLS_self_attn": 0
}