Upload training_config.yml with huggingface_hub

training_config.yml

@@ -5,30 +5,31 @@ model:
   - v_proj
   apply_lora_to_mlp: false
   apply_lora_to_output: false
-  lora_rank: 8
-  lora_alpha: 16
+  lora_rank: 32
+  lora_alpha: 64
   perception_tokens: 2
   use_clip: false
 tokenizer:
   _component_: models.a2a_tokenizer
-  path: checkpoints/Meta-Llama-3-8B-Instruct/original/tokenizer.model
+  path: models/tokenizer.model
 checkpointer:
   _component_: torchtune.utils.FullModelMetaCheckpointer
-  checkpoint_dir: checkpoints/Meta-Llama-3-8B-Instruct/original
+  checkpoint_dir: 
   checkpoint_files:
-  - consolidated.02.pth
+  - 
   adapter_checkpoint: null
   recipe_checkpoint: null
-  output_dir: output_checkpoints/experiment_3
+  output_dir: output_checkpoints/experiment_1
   model_type: LLAMA3
 resume_from_checkpoint: false
-interim_checkpoint_steps: 5000
+interim_checkpoint_steps: 15000
 interim_gen_steps: null
-max_new_tokens: 100
-temperature: 0.6
-top_k: 300
+max_new_tokens: 88
+temperature: 0.7
+top_k: 232
 dataset:
   _component_: ds.EvenBatcher
+  buffer_size: 73
   dataset:
     _component_: ds.RoundRobinDataset
     datasets:
@@ -45,21 +46,21 @@ dataset:
       train_on_input: false
 seed: null
 shuffle: true
-batch_size: 8
+batch_size: 6
 optimizer:
   _component_: torch.optim.AdamW
-  weight_decay: 0.01
-  lr: 0.0003
+  weight_decay: 0.99
+  lr: 20.0e-05
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
-  num_warmup_steps: 100
+  num_warmup_steps: 4
 loss:
   _component_: torch.nn.CrossEntropyLoss
-epochs: 2
-max_steps_per_epoch: 10
-gradient_accumulation_steps: 64
+epochs: 60
+max_steps_per_epoch: null
+gradient_accumulation_steps: 260
 compile: false
-output_dir: /tmp/lora_finetune_output
+output_dir: /workspace/hebbanvogola/lora_finetune_output
 metric_logger:
   _component_: torchtune.utils.metric_logging.DiskLogger
   log_dir: ${output_dir}
@@ -76,8 +77,8 @@ inference:
     {video}
 
     Caption the previous video.'
-  max_new_tokens: 300
-  temperature: 0.6
-  top_k: 300
+  max_new_tokens: 231
+  temperature: 0.8
+  top_k: 231
   quantizer: null
 gradient-accumulation-steps: 32