| [2023-08-11 01:11:49,924] [INFO] [runner.py:555:main] cmd = /home/w32zhong/anaconda3/envs/llamax/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbNCwgNSwgNiwgN119 --master_addr=127.0.0.1 --master_port=8921 --enable_each_rank_log=None train.py --model_name_or_path lmsys/vicuna-13b-v1.5 --data_file ./data/finetune-pairs.json --debug_single_layer False --dryrun False --use_lora False --ctx_length 2048 --datamap_nprocs 10 --use_flash_att2 True --load_8bit False --num_train_epochs 3 --output_dir ./output --save_strategy steps --save_steps 100 --save_total_limit 2 --logging_steps 1 --report_to tensorboard --per_device_train_batch_size 1 --gradient_accumulation_steps 12 --max_grad_norm 1.0 --learning_rate 2e-5 --warmup_ratio 0.03 --fp16 False --bf16 True --deepspeed /tmp/tmp5ncfkvzc | |
| [2023-08-11 01:11:51,958] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) | |
| ===================================BUG REPORT=================================== | |
| Welcome to bitsandbytes. For bug reports, please run | |
| python -m bitsandbytes | |
| and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues | |
| ================================================================================ | |
| CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so | |
| CUDA SETUP: Highest compute capability among GPUs detected: 8.9 | |
| CUDA SETUP: Detected CUDA version 121 | |
| CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_P2P_DISABLE=1 | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_DEBUG=INFO | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_IB_DISABLE=1 | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_BLOCKING_WAIT=1 | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:145:main] WORLD INFO DICT: {'localhost': [4, 5, 6, 7]} | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:151:main] nnodes=1, num_local_procs=4, node_rank=0 | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:162:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0, 1, 2, 3]}) | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:163:main] dist_world_size=4 | |
| [2023-08-11 01:11:53,118] [INFO] [launch.py:165:main] Setting CUDA_VISIBLE_DEVICES=4,5,6,7 | |
| [2023-08-11 01:11:55,214] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) | |
| [2023-08-11 01:11:55,257] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) | |
| [2023-08-11 01:11:55,260] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) | |
| [2023-08-11 01:11:55,265] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) | |
| ===================================BUG REPORT=================================== | |
| Welcome to bitsandbytes. For bug reports, please run | |
| python -m bitsandbytes | |
| and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues | |
| ================================================================================ | |
| CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so | |
| CUDA SETUP: Highest compute capability among GPUs detected: 8.9 | |
| CUDA SETUP: Detected CUDA version 121 | |
| CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... | |
| ===================================BUG REPORT=================================== | |
| Welcome to bitsandbytes. For bug reports, please run | |
| python -m bitsandbytes | |
| and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues | |
| ================================================================================ | |
| CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so | |
| CUDA SETUP: Highest compute capability among GPUs detected: 8.9 | |
| CUDA SETUP: Detected CUDA version 121 | |
| CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... | |
| ===================================BUG REPORT=================================== | |
| Welcome to bitsandbytes. For bug reports, please run | |
| python -m bitsandbytes | |
| and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues | |
| ================================================================================ | |
| CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so | |
| CUDA SETUP: Highest compute capability among GPUs detected: 8.9 | |
| CUDA SETUP: Detected CUDA version 121 | |
| CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... | |
| ===================================BUG REPORT=================================== | |
| Welcome to bitsandbytes. For bug reports, please run | |
| python -m bitsandbytes | |
| and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues | |
| ================================================================================ | |
| CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so | |
| CUDA SETUP: Highest compute capability among GPUs detected: 8.9 | |
| CUDA SETUP: Detected CUDA version 121 | |
| CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... | |
| [2023-08-11 01:11:56,439] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented | |
| [2023-08-11 01:11:56,439] [INFO] [comm.py:616:init_distributed] cdb=None | |
| [2023-08-11 01:11:56,439] [INFO] [comm.py:643:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl | |
| [2023-08-11 01:11:56,544] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented | |
| [2023-08-11 01:11:56,544] [INFO] [comm.py:616:init_distributed] cdb=None | |
| [2023-08-11 01:11:56,607] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented | |
| [2023-08-11 01:11:56,607] [INFO] [comm.py:616:init_distributed] cdb=None | |
| [2023-08-11 01:11:56,608] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented | |
| [2023-08-11 01:11:56,609] [INFO] [comm.py:616:init_distributed] cdb=None | |
| MyArguments(model_name_or_path='lmsys/vicuna-13b-v1.5', data_file='./data/finetune-pairs.json', dryrun=False, ctx_length=2048, datamap_nprocs=10, use_flash_att2=True, use_lora=False, load_8bit=False, cache_dir=None, specified_tokenizer=None, debug_single_layer=False) | |
| TrainingArguments( | |
| _n_gpu=1, | |
| adafactor=False, | |
| adam_beta1=0.9, | |
| adam_beta2=0.999, | |
| adam_epsilon=1e-08, | |
| auto_find_batch_size=False, | |
| bf16=True, | |
| bf16_full_eval=False, | |
| data_seed=None, | |
| dataloader_drop_last=False, | |
| dataloader_num_workers=0, | |
| dataloader_pin_memory=True, | |
| ddp_backend=None, | |
| ddp_broadcast_buffers=None, | |
| ddp_bucket_cap_mb=None, | |
| ddp_find_unused_parameters=None, | |
| ddp_timeout=1800, | |
| debug=[], | |
| deepspeed=/tmp/tmp5ncfkvzc, | |
| disable_tqdm=False, | |
| do_eval=False, | |
| do_predict=False, | |
| do_train=False, | |
| eval_accumulation_steps=None, | |
| eval_delay=0, | |
| eval_steps=None, | |
| evaluation_strategy=no, | |
| fp16=False, | |
| fp16_backend=auto, | |
| fp16_full_eval=False, | |
| fp16_opt_level=O1, | |
| fsdp=[], | |
| fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, | |
| fsdp_min_num_params=0, | |
| fsdp_transformer_layer_cls_to_wrap=None, | |
| full_determinism=False, | |
| gradient_accumulation_steps=12, | |
| gradient_checkpointing=False, | |
| greater_is_better=None, | |
| group_by_length=False, | |
| half_precision_backend=auto, | |
| hub_model_id=None, | |
| hub_private_repo=False, | |
| hub_strategy=every_save, | |
| hub_token=<HUB_TOKEN>, | |
| ignore_data_skip=False, | |
| include_inputs_for_metrics=False, | |
| jit_mode_eval=False, | |
| label_names=None, | |
| label_smoothing_factor=0.0, | |
| learning_rate=2e-05, | |
| length_column_name=length, | |
| load_best_model_at_end=False, | |
| local_rank=0, | |
| log_level=passive, | |
| log_level_replica=warning, | |
| log_on_each_node=True, | |
| logging_dir=./output/runs/Aug11_01-11-56_watgpu-100, | |
| logging_first_step=False, | |
| logging_nan_inf_filter=True, | |
| logging_steps=1.0, | |
| logging_strategy=steps, | |
| lr_scheduler_type=linear, | |
| max_grad_norm=1.0, | |
| max_steps=-1, | |
| metric_for_best_model=None, | |
| mp_parameters=, | |
| no_cuda=False, | |
| num_train_epochs=3.0, | |
| optim=adamw_hf, | |
| optim_args=None, | |
| output_dir=./output, | |
| overwrite_output_dir=False, | |
| past_index=-1, | |
| per_device_eval_batch_size=8, | |
| per_device_train_batch_size=1, | |
| prediction_loss_only=False, | |
| push_to_hub=False, | |
| push_to_hub_model_id=None, | |
| push_to_hub_organization=None, | |
| push_to_hub_token=<PUSH_TO_HUB_TOKEN>, | |
| ray_scope=last, | |
| remove_unused_columns=True, | |
| report_to=['tensorboard'], | |
| resume_from_checkpoint=None, | |
| run_name=./output, | |
| save_on_each_node=False, | |
| save_safetensors=False, | |
| save_steps=100, | |
| save_strategy=steps, | |
| save_total_limit=2, | |
| seed=42, | |
| sharded_ddp=[], | |
| skip_memory_metrics=True, | |
| tf32=None, | |
| torch_compile=False, | |
| torch_compile_backend=None, | |
| torch_compile_mode=None, | |
| torchdynamo=None, | |
| tpu_metrics_debug=False, | |
| tpu_num_cores=None, | |
| use_ipex=False, | |
| use_legacy_prediction_loop=False, | |
| use_mps_device=False, | |
| warmup_ratio=0.03, | |
| warmup_steps=0, | |
| weight_decay=0.0, | |
| xpu_backend=None, | |
| ) | |
| { | |
| "zero_optimization": { | |
| "stage": 3, | |
| "offload_optimizer": { | |
| "device": "cpu", | |
| "pin_memory": true | |
| }, | |
| "overlap_comm": true, | |
| "contiguous_gradients": true, | |
| "sub_group_size": 1000000000.0, | |
| "reduce_bucket_size": "auto", | |
| "stage3_prefetch_bucket_size": "auto", | |
| "stage3_param_persistence_threshold": "auto", | |
| "stage3_max_live_parameters": 1000000000.0, | |
| "stage3_max_reuse_distance": 1000000000.0, | |
| "stage3_gather_16bit_weights_on_model_save": true | |
| }, | |
| "optimizer": { | |
| "type": "AdamW", | |
| "params": { | |
| "betas": [ | |
| 0.9, | |
| 0.999 | |
| ], | |
| "eps": 1e-08, | |
| "weight_decay": 0.0 | |
| } | |
| }, | |
| "scheduler": { | |
| "type": "WarmupLR", | |
| "params": { | |
| "warmup_min_lr": 0, | |
| "warmup_max_lr": 2e-05, | |
| "warmup_num_steps": "auto", | |
| "warmup_type": "linear" | |
| } | |
| }, | |
| "fp16": { | |
| "enabled": false, | |
| "loss_scale": 0, | |
| "initial_scale_power": 16, | |
| "loss_scale_window": 1000, | |
| "hysteresis": 2, | |
| "min_loss_scale": 1 | |
| }, | |
| "bf16": { | |
| "enabled": true | |
| }, | |
| "gradient_accumulation_steps": 12, | |
| "train_micro_batch_size_per_gpu": 1, | |
| "train_batch_size": 48, | |
| "gradient_clipping": 1.0, | |
| "steps_per_print": Infinity, | |
| "wall_clock_breakdown": false | |
| } | |