[2023-08-11 01:11:49,924] [INFO] [runner.py:555:main] cmd = /home/w32zhong/anaconda3/envs/llamax/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbNCwgNSwgNiwgN119 --master_addr=127.0.0.1 --master_port=8921 --enable_each_rank_log=None train.py --model_name_or_path lmsys/vicuna-13b-v1.5 --data_file ./data/finetune-pairs.json --debug_single_layer False --dryrun False --use_lora False --ctx_length 2048 --datamap_nprocs 10 --use_flash_att2 True --load_8bit False --num_train_epochs 3 --output_dir ./output --save_strategy steps --save_steps 100 --save_total_limit 2 --logging_steps 1 --report_to tensorboard --per_device_train_batch_size 1 --gradient_accumulation_steps 12 --max_grad_norm 1.0 --learning_rate 2e-5 --warmup_ratio 0.03 --fp16 False --bf16 True --deepspeed /tmp/tmp5ncfkvzc [2023-08-11 01:11:51,958] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) ===================================BUG REPORT=================================== Welcome to bitsandbytes. For bug reports, please run python -m bitsandbytes and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues ================================================================================ CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so CUDA SETUP: Highest compute capability among GPUs detected: 8.9 CUDA SETUP: Detected CUDA version 121 CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_P2P_DISABLE=1 [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_DEBUG=INFO [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_IB_DISABLE=1 [2023-08-11 01:11:53,118] [INFO] [launch.py:138:main] 0 NCCL_BLOCKING_WAIT=1 [2023-08-11 01:11:53,118] [INFO] [launch.py:145:main] WORLD INFO DICT: {'localhost': [4, 5, 6, 7]} [2023-08-11 01:11:53,118] [INFO] [launch.py:151:main] nnodes=1, num_local_procs=4, node_rank=0 [2023-08-11 01:11:53,118] [INFO] [launch.py:162:main] global_rank_mapping=defaultdict(, {'localhost': [0, 1, 2, 3]}) [2023-08-11 01:11:53,118] [INFO] [launch.py:163:main] dist_world_size=4 [2023-08-11 01:11:53,118] [INFO] [launch.py:165:main] Setting CUDA_VISIBLE_DEVICES=4,5,6,7 [2023-08-11 01:11:55,214] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) [2023-08-11 01:11:55,257] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) [2023-08-11 01:11:55,260] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) [2023-08-11 01:11:55,265] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect) ===================================BUG REPORT=================================== Welcome to bitsandbytes. For bug reports, please run python -m bitsandbytes and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues ================================================================================ CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so CUDA SETUP: Highest compute capability among GPUs detected: 8.9 CUDA SETUP: Detected CUDA version 121 CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... ===================================BUG REPORT=================================== Welcome to bitsandbytes. For bug reports, please run python -m bitsandbytes and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues ================================================================================ CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so CUDA SETUP: Highest compute capability among GPUs detected: 8.9 CUDA SETUP: Detected CUDA version 121 CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... ===================================BUG REPORT=================================== Welcome to bitsandbytes. For bug reports, please run python -m bitsandbytes and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues ================================================================================ CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so CUDA SETUP: Highest compute capability among GPUs detected: 8.9 CUDA SETUP: Detected CUDA version 121 CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... ===================================BUG REPORT=================================== Welcome to bitsandbytes. For bug reports, please run python -m bitsandbytes and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues ================================================================================ CUDA SETUP: CUDA runtime path found: /home/w32zhong/anaconda3/envs/llamax/lib/libcudart.so CUDA SETUP: Highest compute capability among GPUs detected: 8.9 CUDA SETUP: Detected CUDA version 121 CUDA SETUP: Loading binary /home/w32zhong/anaconda3/envs/llamax/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so... [2023-08-11 01:11:56,439] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented [2023-08-11 01:11:56,439] [INFO] [comm.py:616:init_distributed] cdb=None [2023-08-11 01:11:56,439] [INFO] [comm.py:643:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl [2023-08-11 01:11:56,544] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented [2023-08-11 01:11:56,544] [INFO] [comm.py:616:init_distributed] cdb=None [2023-08-11 01:11:56,607] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented [2023-08-11 01:11:56,607] [INFO] [comm.py:616:init_distributed] cdb=None [2023-08-11 01:11:56,608] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented [2023-08-11 01:11:56,609] [INFO] [comm.py:616:init_distributed] cdb=None MyArguments(model_name_or_path='lmsys/vicuna-13b-v1.5', data_file='./data/finetune-pairs.json', dryrun=False, ctx_length=2048, datamap_nprocs=10, use_flash_att2=True, use_lora=False, load_8bit=False, cache_dir=None, specified_tokenizer=None, debug_single_layer=False) TrainingArguments( _n_gpu=1, adafactor=False, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, auto_find_batch_size=False, bf16=True, bf16_full_eval=False, data_seed=None, dataloader_drop_last=False, dataloader_num_workers=0, dataloader_pin_memory=True, ddp_backend=None, ddp_broadcast_buffers=None, ddp_bucket_cap_mb=None, ddp_find_unused_parameters=None, ddp_timeout=1800, debug=[], deepspeed=/tmp/tmp5ncfkvzc, disable_tqdm=False, do_eval=False, do_predict=False, do_train=False, eval_accumulation_steps=None, eval_delay=0, eval_steps=None, evaluation_strategy=no, fp16=False, fp16_backend=auto, fp16_full_eval=False, fp16_opt_level=O1, fsdp=[], fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_min_num_params=0, fsdp_transformer_layer_cls_to_wrap=None, full_determinism=False, gradient_accumulation_steps=12, gradient_checkpointing=False, greater_is_better=None, group_by_length=False, half_precision_backend=auto, hub_model_id=None, hub_private_repo=False, hub_strategy=every_save, hub_token=, ignore_data_skip=False, include_inputs_for_metrics=False, jit_mode_eval=False, label_names=None, label_smoothing_factor=0.0, learning_rate=2e-05, length_column_name=length, load_best_model_at_end=False, local_rank=0, log_level=passive, log_level_replica=warning, log_on_each_node=True, logging_dir=./output/runs/Aug11_01-11-56_watgpu-100, logging_first_step=False, logging_nan_inf_filter=True, logging_steps=1.0, logging_strategy=steps, lr_scheduler_type=linear, max_grad_norm=1.0, max_steps=-1, metric_for_best_model=None, mp_parameters=, no_cuda=False, num_train_epochs=3.0, optim=adamw_hf, optim_args=None, output_dir=./output, overwrite_output_dir=False, past_index=-1, per_device_eval_batch_size=8, per_device_train_batch_size=1, prediction_loss_only=False, push_to_hub=False, push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=, ray_scope=last, remove_unused_columns=True, report_to=['tensorboard'], resume_from_checkpoint=None, run_name=./output, save_on_each_node=False, save_safetensors=False, save_steps=100, save_strategy=steps, save_total_limit=2, seed=42, sharded_ddp=[], skip_memory_metrics=True, tf32=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, torchdynamo=None, tpu_metrics_debug=False, tpu_num_cores=None, use_ipex=False, use_legacy_prediction_loop=False, use_mps_device=False, warmup_ratio=0.03, warmup_steps=0, weight_decay=0.0, xpu_backend=None, ) { "zero_optimization": { "stage": 3, "offload_optimizer": { "device": "cpu", "pin_memory": true }, "overlap_comm": true, "contiguous_gradients": true, "sub_group_size": 1000000000.0, "reduce_bucket_size": "auto", "stage3_prefetch_bucket_size": "auto", "stage3_param_persistence_threshold": "auto", "stage3_max_live_parameters": 1000000000.0, "stage3_max_reuse_distance": 1000000000.0, "stage3_gather_16bit_weights_on_model_save": true }, "optimizer": { "type": "AdamW", "params": { "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0 } }, "scheduler": { "type": "WarmupLR", "params": { "warmup_min_lr": 0, "warmup_max_lr": 2e-05, "warmup_num_steps": "auto", "warmup_type": "linear" } }, "fp16": { "enabled": false, "loss_scale": 0, "initial_scale_power": 16, "loss_scale_window": 1000, "hysteresis": 2, "min_loss_scale": 1 }, "bf16": { "enabled": true }, "gradient_accumulation_steps": 12, "train_micro_batch_size_per_gpu": 1, "train_batch_size": 48, "gradient_clipping": 1.0, "steps_per_print": Infinity, "wall_clock_breakdown": false }