compress pretrain dataset

scripts/prepare_pretrain_dataset.py

@@ -360,7 +360,7 @@ datasets_configs = [
     # {'path': 'keivalya/MedQuad-MedicalQnADataset', 'split': 'train', 'format': '{Question} {Answer}'},
     # {'path': 'NousResearch/CharacterCodex', 'split': 'train', 'format': '{scenario} {description}'},
     # {'path': 'nampdn-ai/tiny-textbooks', 'split': 'train+test', 'format': '{textbook}'},
-    
+
     # code
     # {'path': 'nampdn-ai/tiny-codes', 'split': 'train[:5%]', 'format': '{prompt} {response}'},
     *[
@@ -421,4 +421,5 @@ outputs = optimize(
     # Number of tokens to store by chunks. This is roughly 64MB of tokens per chunk.
     chunk_size=(2049 * 8012),
     num_workers=32,
+    compression='zstd',
 )