optimizer:
  allreduce_bucket_size: 500000000
  beta1: 0.9
  beta2: 0.95
  debug_log: false
  eps: 1e-08
  gradient_clipping: 0.0
  zero: true
  zero_save_static: false
topology:
  activation_checkpointing_type: disabled
  global_batch_size: 1024
  gradient_accumulation_steps: 4
  micro_batch_size: 2
  model_parallel_size: 1
  pipe_parallel_size: 2
  pipe_partition_method: balanced
  pipe_partition_overwrite: null
  sequence_parallel: false
trainer:
  seed: 42
  train_iterations: 72000
training:
  allow_missing_params_in_optimizer: true
training_groups:
- group_name: param_group
  independent_weight_decay: true
  learning_rate_scheduler:
    learning_rate: 11.313708498984761
    learning_rate_decay_iters: 72000
    learning_rate_decay_style: cosine
    learning_rate_minimum: 1.131370849898476
    learning_rate_warmup_steps: 500
  parameters_exclude:
  - norm
  weight_decay: 0.0001221
transformer_architecture:
  attention_bias: false
  attention_num_kv_heads: null
  attention_qkv_in_one: true
  dropout_after_attention: 0.0
  dropout_after_mlp: 0.0
  dropout_attention_probs: 0.0
  dropout_embedding: 0.0
  dropout_image_encoder: 0.0
  hidden_size: 4096
  image_encoder: false
  key_query_norm: false
  layernorm:
    layernorm_epsilon: 1e-05
    optimization_type: torch
  local_attention_window_size: null
  masked_softmax:
    kernel: flash_attention
    scale: 1.0
    softmax_in_fp32: false
  mlp_bias: false
  mlp_factor: 2.66796875
  mlp_type: swiglu
  norm_type: rms
  num_attention_heads: 32
  num_layers: 32
  num_local_attention_heads: 0
  precision: bfloat16
  relative_position_embedding_type: rotary_complex
  reset_attention_mask: false
  reset_position_ids: false
  rotary_embedding_base: 10000
  rotary_percentage: 1.0
  sequence_length: 4096
  umup:
    act_mult: 1.0
    attn_mult: 1.0
    enable: true
    loss_mult: 1.0
    normalize_depth_to_num_layers: true
    residual_attn_ratio: 0.25
    residual_mult: 1.0
  vocab_file: null
  vocab_size: 65536
  weight_tying: false