Model save

run.log

@@ -6184,3 +6184,199 @@ Time to load utils op: 0.0003948211669921875 seconds
 [2022-12-17 14:30:54,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
 [2022-12-17 14:30:54,828] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt
 [2022-12-17 14:30:54,828] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5000 is ready now!
+[2022-12-17 14:33:04,234] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.7, git-hash=unknown, git-branch=unknown
+[2022-12-17 14:33:04,295] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False
+[2022-12-17 14:33:05,834] [WARNING] [cpu_adam.py:83:__init__] FP16 params for CPUAdam may not work on AMD CPUs
+Installed CUDA version 11.6 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Time to load cpu_adam op: 3.3625645637512207 seconds
+Adam Optimizer #1 is created with AVX2 arithmetic capability.
+Config: alpha=0.000010, betas=(0.900000, 0.999000), weight_decay=0.000000, adam_w=1
+[2022-12-17 14:33:09,860] [INFO] [logging.py:68:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adamw as basic optimizer
+[2022-12-17 14:33:10,163] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam
+[2022-12-17 14:33:10,163] [INFO] [utils.py:52:is_zero_supported_optimizer] Checking ZeRO support for optimizer=DeepSpeedCPUAdam type=<class 'deepspeed.ops.adam.cpu_adam.DeepSpeedCPUAdam'>
+[2022-12-17 14:33:10,163] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 ZeRO stage 2 optimizer
+[2022-12-17 14:33:10,163] [INFO] [stage_1_and_2.py:140:__init__] Reduce bucket size 200000000
+[2022-12-17 14:33:10,163] [INFO] [stage_1_and_2.py:141:__init__] Allgather bucket size 200000000
+[2022-12-17 14:33:10,163] [INFO] [stage_1_and_2.py:142:__init__] CPU Offload: True
+[2022-12-17 14:33:10,163] [INFO] [stage_1_and_2.py:143:__init__] Round robin gradient partitioning: False
+Time to load utils op: 0.0004093647003173828 seconds
+Rank: 0 partition count [1] and sizes[(1543304960, False)] 
+[2022-12-17 14:33:14,433] [INFO] [utils.py:827:see_memory_usage] Before initializing optimizer states
+[2022-12-17 14:33:14,434] [INFO] [utils.py:828:see_memory_usage] MA 6.0 GB         Max_MA 19.53 GB         CA 29.61 GB         Max_CA 30 GB 
+[2022-12-17 14:33:14,434] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 48.37 GB, percent = 24.6%
+[2022-12-17 14:33:20,694] [INFO] [utils.py:827:see_memory_usage] After initializing optimizer states
+[2022-12-17 14:33:20,694] [INFO] [utils.py:828:see_memory_usage] MA 6.0 GB         Max_MA 6.0 GB         CA 29.61 GB         Max_CA 30 GB 
+[2022-12-17 14:33:20,695] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 68.22 GB, percent = 34.7%
+[2022-12-17 14:33:20,695] [INFO] [stage_1_and_2.py:525:__init__] optimizer state initialized
+[2022-12-17 14:33:20,771] [INFO] [utils.py:827:see_memory_usage] After initializing ZeRO optimizer
+[2022-12-17 14:33:20,772] [INFO] [utils.py:828:see_memory_usage] MA 6.0 GB         Max_MA 6.0 GB         CA 29.61 GB         Max_CA 30 GB 
+[2022-12-17 14:33:20,772] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 68.22 GB, percent = 34.7%
+[2022-12-17 14:33:20,801] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = adamw
+[2022-12-17 14:33:20,801] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using configured LR scheduler = WarmupDecayLR
+[2022-12-17 14:33:20,801] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = <deepspeed.runtime.lr_schedules.WarmupDecayLR object at 0x7f5f21a66970>
+[2022-12-17 14:33:20,801] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[1e-05], mom=[[0.9, 0.999]]
+[2022-12-17 14:33:20,803] [INFO] [config.py:1020:print] DeepSpeedEngine configuration:
+[2022-12-17 14:33:20,803] [INFO] [config.py:1024:print]   activation_checkpointing_config  {
+    "partition_activations": false, 
+    "contiguous_memory_optimization": false, 
+    "cpu_checkpointing": false, 
+    "number_checkpoints": null, 
+    "synchronize_checkpoint_boundary": false, 
+    "profile": false
+}
+[2022-12-17 14:33:20,803] [INFO] [config.py:1024:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}
+[2022-12-17 14:33:20,803] [INFO] [config.py:1024:print]   amp_enabled .................. False
+[2022-12-17 14:33:20,803] [INFO] [config.py:1024:print]   amp_params ................... False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   autotuning_config ............ {
+    "enabled": false, 
+    "start_step": null, 
+    "end_step": null, 
+    "metric_path": null, 
+    "arg_mappings": null, 
+    "metric": "throughput", 
+    "model_info": null, 
+    "results_dir": "autotuning_results", 
+    "exps_dir": "autotuning_exps", 
+    "overwrite": true, 
+    "fast": true, 
+    "start_profile_step": 3, 
+    "end_profile_step": 5, 
+    "tuner_type": "gridsearch", 
+    "tuner_early_stopping": 5, 
+    "tuner_num_trials": 50, 
+    "model_info_path": null, 
+    "mp_size": 1, 
+    "max_train_batch_size": null, 
+    "min_train_batch_size": 1, 
+    "max_train_micro_batch_size_per_gpu": 1.024000e+03, 
+    "min_train_micro_batch_size_per_gpu": 1, 
+    "num_tuning_micro_batch_sizes": 3
+}
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   bfloat16_enabled ............. False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   checkpoint_parallel_write_pipeline  False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   checkpoint_tag_validation_enabled  True
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   checkpoint_tag_validation_fail  False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f5ddadcccd0>
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   communication_data_type ...... None
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   curriculum_enabled ........... False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   curriculum_params ............ False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   dataloader_drop_last ......... False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   disable_allgather ............ False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   dump_state ................... False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   dynamic_loss_scale_args ...... {'init_scale': 65536, 'scale_window': 1000, 'delayed_shift': 2, 'min_scale': 1}
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   eigenvalue_enabled ........... False
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   eigenvalue_gas_boundary_resolution  1
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   eigenvalue_layer_name ........ bert.encoder.layer
+[2022-12-17 14:33:20,804] [INFO] [config.py:1024:print]   eigenvalue_layer_num ......... 0
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   eigenvalue_max_iter .......... 100
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   eigenvalue_stability ......... 1e-06
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   eigenvalue_tol ............... 0.01
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   eigenvalue_verbose ........... False
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   elasticity_enabled ........... False
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   flops_profiler_config ........ {
+    "enabled": false, 
+    "profile_step": 1, 
+    "module_depth": -1, 
+    "top_modules": 1, 
+    "detailed": true, 
+    "output_file": null
+}
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   fp16_auto_cast ............... False
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   fp16_enabled ................. True
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   fp16_master_weights_and_gradients  False
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   global_rank .................. 0
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   grad_accum_dtype ............. None
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   gradient_accumulation_steps .. 2
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   gradient_clipping ............ 1.0
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   gradient_predivide_factor .... 1.0
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   initial_dynamic_scale ........ 65536
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   load_universal_checkpoint .... False
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   loss_scale ................... 0
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   memory_breakdown ............. False
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   monitor_config ............... <deepspeed.monitor.config.DeepSpeedMonitorConfig object at 0x7f5ddadcc790>
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   nebula_config ................ {
+    "enabled": false, 
+    "persistent_storage_path": null, 
+    "persistent_time_interval": 100, 
+    "num_of_version_in_retention": 2, 
+    "enable_nebula_load": true, 
+    "load_path": null
+}
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   optimizer_legacy_fusion ...... False
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   optimizer_name ............... adamw
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   optimizer_params ............. {'lr': 1e-05, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0}
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}
+[2022-12-17 14:33:20,805] [INFO] [config.py:1024:print]   pld_enabled .................. False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   pld_params ................... False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   prescale_gradients ........... False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   scheduler_name ............... WarmupDecayLR
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   scheduler_params ............. {'last_batch_iteration': -1, 'total_num_steps': 5000, 'warmup_min_lr': 0, 'warmup_max_lr': 1e-05, 'warmup_num_steps': 500}
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   sparse_attention ............. None
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   sparse_gradients_enabled ..... False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   steps_per_print .............. 10
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   train_batch_size ............. 64
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   train_micro_batch_size_per_gpu  32
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   use_node_local_storage ....... False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   wall_clock_breakdown ......... False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   world_size ................... 1
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   zero_allow_untested_optimizer  False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=200000000 allgather_partitions=True allgather_bucket_size=200000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=True, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   zero_enabled ................. True
+[2022-12-17 14:33:20,806] [INFO] [config.py:1024:print]   zero_optimization_stage ...... 2
+[2022-12-17 14:33:20,806] [INFO] [config.py:1009:print_user_config]   json = {
+    "fp16": {
+        "enabled": true, 
+        "loss_scale": 0, 
+        "loss_scale_window": 1000, 
+        "initial_scale_power": 16, 
+        "hysteresis": 2, 
+        "min_loss_scale": 1
+    }, 
+    "optimizer": {
+        "type": "AdamW", 
+        "params": {
+            "lr": 1e-05, 
+            "betas": [0.9, 0.999], 
+            "eps": 1e-08, 
+            "weight_decay": 0.0
+        }
+    }, 
+    "scheduler": {
+        "type": "WarmupDecayLR", 
+        "params": {
+            "last_batch_iteration": -1, 
+            "total_num_steps": 5.000000e+03, 
+            "warmup_min_lr": 0, 
+            "warmup_max_lr": 1e-05, 
+            "warmup_num_steps": 500
+        }
+    }, 
+    "zero_optimization": {
+        "stage": 2, 
+        "offload_optimizer": {
+            "device": "cpu", 
+            "pin_memory": true
+        }, 
+        "allgather_partitions": true, 
+        "allgather_bucket_size": 2.000000e+08, 
+        "overlap_comm": true, 
+        "reduce_scatter": true, 
+        "reduce_bucket_size": 2.000000e+08, 
+        "contiguous_gradients": true
+    }, 
+    "gradient_accumulation_steps": 2, 
+    "gradient_clipping": 1.0, 
+    "train_batch_size": 64, 
+    "train_micro_batch_size_per_gpu": 32
+}
+Time to load utils op: 0.00031685829162597656 seconds
+[2022-12-17 14:33:20,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt...
+[2022-12-17 14:33:21,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt.
+[2022-12-17 14:33:21,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt...
+[2022-12-17 14:33:23,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt.
+[2022-12-17 14:33:23,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2022-12-17 14:33:29,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2022-12-17 14:33:29,392] [INFO] [engine.py:2900:_get_all_zero_checkpoint_state_dicts] successfully read 1 ZeRO state_dicts for rank 0
+[2022-12-17 14:33:32,457] [INFO] [engine.py:2840:_load_zero_checkpoint] loading 1 zero partition checkpoints for rank 0
+{'train_runtime': 67979.462, 'train_samples_per_second': 4.707, 'train_steps_per_second': 0.074, 'train_loss': 0.015194951736927033, 'epoch': 21.19}

runs/Dec16_19-20-02_129-146-123-136/events.out.tfevents.1671219633.129-146-123-136.2641037.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c017ef676235e0740e0a0618b286fdb2beb83053f99d55ccdabeefb3dcd413d4
-size 37257
+oid sha256:01976f155798a1a82260f5d85293172ff196660912b08334fad53806f31f0a1a
+size 37611