pretrain model, extend from 5 to 8 epochs

scripts/TRAIN.md

@@ -46,9 +46,19 @@ save_file(state_dict, 'out/converted_model/model.safetensors')
 ## Evaluate
 
 ```bash
-litgpt evaluate --tasks 'leaderboard' --out_dir 'evaluate-0/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+litgpt evaluate --tasks 'hellaswag,gsm8k,truthfulqa_mc2,mmlu,winogrande,arc_challenge' --out_dir 'evaluate-quick/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
 
-litgpt evaluate --tasks 'hellaswag,gsm8k,truthfulqa_mc2,mmlu,winogrande,arc_challenge' --out_dir 'evaluate-1/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+litgpt evaluate --tasks 'leaderboard' --out_dir 'evaluate-leaderboard/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
 
-litgpt evaluate --tasks 'mmlu_pro,ifeval,mgsm_direct,mathqa,gpqa' --out_dir 'evaluate-2/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+litgpt evaluate --tasks 'bbh_zeroshot,bbh_fewshot,bbh_cot_fewshot,bbh_cot_zeroshot' --out_dir 'evaluate-bigbenchhard/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+
+litgpt evaluate --tasks 'mmlu,mmlu_pro' --out_dir 'evaluate-mmlu/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+
+litgpt evaluate --tasks 'arc_challenge,boolq,gpqa,hellaswag,openbookqa,piqa,siqa,truthfulqa_mc2,winogrande' --out_dir 'evaluate-reasoning/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+
+litgpt evaluate --tasks 'mmlu_multilingual,mgsm' --out_dir 'evaluate-multilinguals/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+
+litgpt evaluate --tasks 'gsm8k,mathqa' --out_dir 'evaluate-math/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
+
+litgpt evaluate --tasks 'qasper' --out_dir 'evaluate-long/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
 ```

scripts/pretrain-model.yaml

@@ -57,7 +57,7 @@ data:
 # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
 train:
   # Number of optimizer steps between saving checkpoints (type: Optional[int], default: 1000)
-  save_interval: 500
+  save_interval: 200
 
   # Number of iterations between logging calls (type: int, default: 1)
   log_interval: 1
@@ -77,7 +77,8 @@ train:
 
   # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
   # max_tokens: 3000000000000
-  max_tokens: 8159107755 # 796399 * 2049 * 5
+  # max_tokens: 8159107755 # 796399 * 2049 * 5
+  max_tokens: 13054572408 # 796399 * 2049 * 8
 
   # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
   max_steps:
@@ -120,10 +121,12 @@ optimizer:
   
   init_args:
     #   (type: float, default: 0.001)
-    lr: 1e-3
+    # lr: 1e-3
+    lr: 1e-4
     
     #   (type: float, default: 0.01)
-    weight_decay: 0.01
+    # weight_decay: 0.01
+    weight_decay: 0.1
     
     #   (type: tuple, default: (0.9,0.999))
     betas: