contrain model

scripts/TRAIN.md

@@ -41,17 +41,17 @@ litgpt pretrain --config ./pretrain-model.yaml
 ```
 
 ```bash
-litgpt convert_from_litgpt out/pretrain/final/ out/converted_model
+litgpt convert_from_litgpt out/pretrain/final/ out/converted_pretrain
 cp config.json out/pretrain/final/
-cp config.json out/converted_model/
+cp config.json out/converted_pretrain/
 ```
 
 ```python
 import torch
 from safetensors.torch import save_file
 
-state_dict = torch.load('out/converted_model/model.pth', map_location='cpu')
-save_file(state_dict, 'out/converted_model/model.safetensors')
+state_dict = torch.load('out/converted_pretrain/model.pth', map_location='cpu')
+save_file(state_dict, 'out/converted_pretrain/model.safetensors')
 ```
 
 ### Continued Pretraining

scripts/contrain-model.yaml

@@ -85,7 +85,7 @@ train:
   max_steps:
 
   # Limits the length of samples. Off by default (type: Optional[int], default: null)
-  max_seq_length: 8193
+  max_seq_length:
 
   # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
   tie_embeddings: true