| accum_freq: 1 | |
| attn_activation: None | |
| attn_name: auto | |
| attn_seq_scalar: None | |
| attn_seq_scalar_alpha: None | |
| average: None | |
| average_coefficients: None | |
| beta1: 0.9 | |
| beta2: 0.95 | |
| checkpoint_path: /admin/home-sy/dcnlp_logs/c4_original-d=96_l=8_h=4-1.0/checkpoints | |
| copy_codebase: False | |
| data_key: txt | |
| dataset_manifest: None | |
| dataset_resampled: False | |
| dataset_type: auto | |
| ddp_static_graph: False | |
| debug: False | |
| delete_previous_checkpoint: True | |
| device: cuda:0 | |
| disable_buffer: False | |
| dist_backend: nccl | |
| dist_url: env:// | |
| distill_model: None | |
| distill_pretrained: None | |
| distributed: True | |
| epochs: 5 | |
| epochs_cooldown: None | |
| eps: 1e-08 | |
| experimental_meta_device: False | |
| ffn_type: swiglu | |
| force_distributed: False | |
| force_min_lr: 0.0 | |
| fsdp: False | |
| fsdp_amp: False | |
| fsdp_backward_prefetch: False | |
| fsdp_checkpoint: False | |
| fsdp_cpu_offload: False | |
| fsdp_hybrid: False | |
| fsdp_hybrid_o2: False | |
| fsdp_limit_all_gathers: False | |
| fsdp_pure_bf16: False | |
| fsdp_use_orig_params: False | |
| global_batch_size: 128 | |
| global_val_batch_size: 128 | |
| grad_checkpointing: False | |
| grad_clip_norm: 1.0 | |
| hf_fsdp_block: None | |
| hf_model: None | |
| hf_seq_len: None | |
| ignore_parse_errors: False | |
| load_pretrained_state: False | |
| local_rank: 0 | |
| log_every_n_steps: 20 | |
| log_level: 20 | |
| log_local: False | |
| log_logit_mean: False | |
| log_path: /admin/home-sy/dcnlp_logs/c4_original-d=96_l=8_h=4-1.0/out.log | |
| logs: /admin/home-sy/dcnlp_logs | |
| lr: 0.003 | |
| lr_cooldown_end: 3e-05 | |
| lr_cooldown_power: 1.0 | |
| lr_scheduler: cosine | |
| model: d=96_l=8_h=4 | |
| model_norm: gain_only_lp_layer_norm | |
| moe_capacity_factor: 1.25 | |
| moe_expert_model_parallelism: False | |
| moe_freq: 0 | |
| moe_loss_weight: 0.1 | |
| moe_num_experts: None | |
| moe_top_k: 2 | |
| moe_weight_parallelism: False | |
| multiple_data_passes: False | |
| name: c4_original-d=96_l=8_h=4-1.0 | |
| no_set_device_rank: False | |
| optimizer: adamw | |
| per_gpu_batch_size: 16 | |
| per_gpu_val_batch_size: 16 | |
| positional_embedding_type: rotary | |
| precision: amp_bfloat16 | |
| pretrained: None | |
| qk_norm: True | |
| rank: 0 | |
| remote_sync: s3://dcnlp-west/dcnlp_experiments_v3 | |
| remote_sync_frequency: 300 | |
| remote_sync_protocol: s3 | |
| report_to: | |
| resume: s3://dcnlp-west/dcnlp_experiments_v3/c4_original-d=96_l=8_h=4-1.0/checkpoints/epoch_1.pt | |
| save_frequency: 1 | |
| save_most_recent: False | |
| seed: 124 | |
| seq_len: 2048 | |
| skip_scheduler: False | |
| squash_mask_left: True | |
| target_mask_individual: 50400 | |
| target_mask_left: 50300 | |
| tensorboard: False | |
| tensorboard_path: | |
| torchcompile: False | |
| torchscript: False | |
| trace: False | |
| train_data: None | |
| train_data_mix_weights: None | |
| train_data_upsampling_factors: None | |
| train_num_samples: None | |
| use_bn_sync: False | |
| use_bnb_linear: None | |
| val_data: ['training/eval_data/val_tok_mult/openlm/shard_00000000.tar', 'training/eval_data/c4_val/shard-{0000000..0000010}.tar', 'training/eval_data/val_tok_mult/paloma_4chan_meta_sep/00000001.tar', 'training/eval_data/val_tok_mult/paloma_c4_100_domains/00000001.tar', 'training/eval_data/val_tok_mult/paloma_c4_en/00000001.tar', 'training/eval_data/val_tok_mult/paloma_dolma-v1_5/00000001.tar', 'training/eval_data/val_tok_mult/paloma_dolma_100_programing_languages/00000001.tar', 'training/eval_data/val_tok_mult/paloma_dolma_100_subreddits/00000001.tar', 'training/eval_data/val_tok_mult/paloma_falcon-refinedweb/00000001.tar', 'training/eval_data/val_tok_mult/paloma_gab/00000001.tar', 'training/eval_data/val_tok_mult/paloma_m2d2_s2orc_unsplit_dedup/00000001.tar', 'training/eval_data/val_tok_mult/paloma_m2d2_wikipedia_unsplit/00000001.tar', 'training/eval_data/val_tok_mult/paloma_manosphere_meta_sep/00000001.tar', 'training/eval_data/val_tok_mult/paloma_mc4/00000001.tar', 'training/eval_data/val_tok_mult/paloma_ptb/00000001.tar', 'training/eval_data/val_tok_mult/paloma_redpajama/00000001.tar', 'training/eval_data/val_tok_mult/paloma_twitterAAE_HELM_fixed/00000001.tar', 'training/eval_data/val_tok_mult/paloma_wikitext_103/00000001.tar', '/admin/home-sy/dcnlp/training/eval_data/mmlu/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/hellaswag/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/jeopardy_all/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/triviaqa_sm_sub/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/gsm8k/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/agi_eval_sat_math/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/aqua/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/svamp/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_qa_wikidata/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/arc_easy/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/arc_challenge/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_misconceptions/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/copa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/siqa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/commonsense_qa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/piqa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/openbook_qa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_novel_concepts/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_strange_stories/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_strategy_qa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/lambada_openai/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/winograd_wsc/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/winogrande/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_conlang_translation/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_language_identification/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_conceptual_combinations/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_elementary_math_qa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_dyck_languages/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/agi_eval_lsat_ar/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_cs_algorithms/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_logical_deduction/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_operators/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_repeat_copy_logic/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/simple_arithmetic_nospaces/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/simple_arithmetic_withspaces/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/math_qa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/logi_qa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/pubmed_qa_labeled/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/squad/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/agi_eval_lsat_rc/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/agi_eval_lsat_lr/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/coqa/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bigbench_understanding_fables/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/boolq/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/agi_eval_sat_en/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/winogender_mc_female/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/winogender_mc_male/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/enterprise_pii_classification/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/bbq/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/human_eval_return_complex/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/human_eval_return_simple/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/human_eval-0.5/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/human_eval-0.25/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/human_eval-0.75/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/human_eval/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/processed_human_eval_cpp/shard-0000000.tar', '/admin/home-sy/dcnlp/training/eval_data/processed_human_eval_js/shard-0000000.tar'] | |
| val_data_key: ['json', 'txt', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'json.gz', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt', 'txt'] | |
| val_frequency: 5 | |
| val_iter_ci: 10000 | |
| val_max_pop_ci: 300000 | |
| val_num_samples: None | |
| val_seq_ci: True | |
| val_tok_ci: True | |
| vocab_size: 50432 | |
| wandb: False | |
| wandb_notes: | |
| wandb_project_name: open-lm | |
| warmup: 100 | |
| wd: 0.033 | |
| workers: 2 | |
| world_size: 8 | |
| z_loss_coefficient: 0.0001 | |