Upload folder using huggingface_hub

models/random_1b/config.yaml

@@ -16,18 +16,18 @@ update_reference: false
 learner_model: null
 train_online: false
 fix_reference: false
-run_name: olmo_32180564_4
+run_name: olmo_33178628_1
 seed: 0
 epoch: null
 dry_run: false
 model:
-  d_model: 1024
-  n_heads: 16
+  d_model: 2048
+  n_heads: 32
   n_kv_heads: null
   clip_qkv: null
-  n_layers: 12
+  n_layers: 24
   mlp_ratio: 4
-  mlp_hidden_size: 4096
+  mlp_hidden_size: 8192
   activation_type: gelu
   block_type: sequential
   block_group_size: 1
@@ -255,7 +255,7 @@ data:
   label_mask_paths: null
   pad_direction: right
   generate_attention_mask: false
-  num_workers: 0
+  num_workers: 16
   drop_last: true
   pin_memory: true
   prefetch_factor: 16
@@ -265,6 +265,7 @@ data:
   extra_data_paths: null
   extra_data_key: null
   load_extra_data_to_ram: false
+  index_path: null
 restore_dataloader: true
 fast_forward_batches: null
 evaluators:
@@ -288,6 +289,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -312,6 +314,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -342,6 +345,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -364,6 +368,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -386,6 +391,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -408,6 +414,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -430,6 +437,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -452,6 +460,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -474,6 +483,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -496,6 +506,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -518,6 +529,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: null
   sft_use_label: false
@@ -540,6 +552,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -562,6 +575,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -584,6 +598,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -606,6 +621,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -628,6 +644,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -650,6 +667,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -672,6 +690,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -694,6 +713,7 @@ evaluators:
     extra_data_paths: null
     extra_data_key: null
     load_extra_data_to_ram: false
+    index_path: null
   device_eval_batch_size: null
   subset_num_batches: 1000
   sft_use_label: false
@@ -702,7 +722,7 @@ eval_interval: 2000
 tokenizer:
   identifier: allenai/eleuther-ai-gpt-neox-20b-pii-special
   truncate_direction: right
-save_folder: /n/vast-scratch/kempner_sham_lab/data-olmo/ckpts/32180564_4/
+save_folder: /n/holyscratch01/sham_lab/data-olmo/ckpts/33178628_1/
 remote_save_folder: null
 canceled_check_interval: 50
 save_interval: 5000
@@ -713,7 +733,7 @@ save_num_unsharded_checkpoints_to_keep: 1
 save_overwrite: true
 force_save_unsharded: false
 no_pre_train_checkpoint: false
-load_path: null
+load_path: /n/holyscratch01/sham_lab/data-olmo/ckpts/32845610_1/step105000
 load_path_sharded_checkpointer: null
 reset_optimizer_state: false
 reset_trainer_state: false
@@ -721,8 +741,8 @@ sharded_checkpointer: torch_legacy
 new_style_checkpoints: null
 max_duration: 192000
 global_train_batch_size: 256
-device_train_batch_size: 256
-device_train_microbatch_size: 128
+device_train_batch_size: 64
+device_train_microbatch_size: 32
 device_eval_batch_size: 64
 eval_subset_num_batches: 100
 eval_on_load: false
@@ -733,8 +753,8 @@ precision: amp_bf16
 wandb:
   project: data-olmo
   entity: harvardml
-  group: pretrain-150-9
-  name: olmo_32180564_4
+  group: pretrain-150-1b-1-test
+  name: olmo_33178628_1
   tags:
   - watching
   log_artifacts: false
@@ -756,7 +776,7 @@ fsdp:
   precision: mixed
   hybrid_sharding_num_model_replicas: null
 softmax_auxiliary_loss: true
-time_limit: 210000.0
+time_limit: 2100000.0
 extra_steps_after_cancel: 10
 early_stopping_factor: null
 save_data_indices: true

models/random_1b/model.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7845427fd34bb744773b5bd005b61960a176c5fbfb0254b3f1d94882348f292
-size 1016309727
+oid sha256:863777eca7cd7dab0a013edb4469e24de6af6dc806dea3939f73bda271eaa71a
+size 5656891654

models/random_1b/optim.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8fbb2627c922cb0dd673e13f4ba9d487f3b0ca1abd70c7674cafc2f2ef43e3a0
-size 2032626246
+oid sha256:dd5cff86b087843aab6600528ed375274c616e8d40853b1099221811c0a70ae1
+size 11313806634

models/random_1b/train.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b15d20b2e3f494f0cf5b5c53be57b9bb618477202d15fe3e2ce3a5ead7465a8e
-size 14668
+oid sha256:2ac8c6b6f9f76b9fd9c21a898d148e291f91f17c5d919aa1b6d2327233d96c2a
+size 14604