config_name: "JackFram/llama-68m"
tokenizer_name: "JackFram/llama-68m"
validation_split_percentage: 2
train_file: "/home/dshteyma/target_draft_coupling_code/dataset_dict.json"
dataset_name_local: "RedPajama"
dataset_name: "togethercomputer/RedPajama-Data-1T-Sample"
dataset_name_hub: "togethercomputer/RedPajama-Data-1T-Sample"
# max_train_samples: 1000
# max_eval_samples: 10
do_train: True
do_eval: True
output_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
overwrite_output_dir: True
per_device_train_batch_size: 4
gradient_accumulation_steps: 3
report_to: "tensorboard"
logging_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
logging_steps: 10000
save_steps: 10000
eval_strategy: "steps"
eval_steps: 10000
learning_rate: 0.0001
weight_decay: 0.01
warmup_ratio: 0.05
push_to_hub: False
hub_model_id: "DorinSht/llama_68M_redpajama"
hub_strategy: "all_checkpoints"