pretrain model

.gitignore

@@ -163,4 +163,6 @@ cython_debug/
 .DS_Store
 .ruff_cache
 venv*/
-data/
+data/
+pretrain-data/
+contrain-data/

scripts/prepare_pretrain_dataset.py

@@ -233,7 +233,7 @@ datasets_names = [
 outputs = optimize(
     fn=partial(tokenize_fn, tokenizer=Tokenizer('..')),
     inputs=datasets_names,
-    output_dir='../data/',
+    output_dir='../pretrain-data/',
     # Number of tokens to store by chunks. This is roughly 64MB of tokens per chunk.
     chunk_size=(4097 * 4006),
     num_workers=16,

scripts/pretrain-model.yaml

@@ -46,7 +46,7 @@ data:
   class_path: LitData
 
   init_args:
-    data_path: "../data/"
+    data_path: "../pretrain-data/"
     num_workers: 16
 
 # Training-related arguments. See ``litgpt.args.TrainArgs`` for details