Training in progress, step 200

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a46981ce6221b1fadfa12e0fcff9c19795080609c0d1fbd4b04c3da4ce91de7
+oid sha256:5c85dc03e28785dd14c969e1fd7710e47eed853f65471606d63962f3aca2deca
 size 261131840

runs/Mar13_11-36-50_7acbe5a74b63/events.out.tfevents.1710329812.7acbe5a74b63.337.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b4536f710ec5f31c613e16e54f1110a95aa26b64d708c2b502b187a3c5224b1
-size 9511
+oid sha256:3366f9d7be1e4fcd2b162a9be26cb898efd272d065cbdae09ae10c4ad8728a6d
+size 9722

wandb/debug-internal.log

@@ -1384,3 +1384,51 @@
 2024-03-13 12:05:26,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
 2024-03-13 12:05:26,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
 2024-03-13 12:05:26,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:05:28,258 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:33,258 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:38,259 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:41,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:05:41,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:05:41,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:05:44,215 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:49,216 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:52,358 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 12:05:54,359 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:56,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:05:56,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:05:56,162 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:00,275 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:05,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:10,277 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:11,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:11,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:11,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:16,275 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:21,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:22,359 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 12:06:26,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:26,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:26,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:27,275 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:32,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:37,277 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:41,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:41,123 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:41,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:42,295 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:46,399 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 12:06:46,400 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 12:06:46,400 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 12:06:46,400 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 12:06:46,432 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 12:06:47,401 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:50,500 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 12:06:50,501 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 12:06:50,501 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 12:06:50,502 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 12:06:51,434 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 12:06:52,359 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 12:06:53,360 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:56,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:56,123 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:56,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages

wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.1275, "train/grad_norm": 0.48586273193359375, "train/learning_rate": 6.54164563305465e-07, "train/epoch": 15.2, "train/global_step": 190, "_timestamp": 1710331517.486697, "_runtime": 2694.1046879291534, "_step": 22, "train/train_runtime": 361.3918, "train/train_samples_per_second": 1.328, "train/train_steps_per_second": 0.083, "train/total_flos": 1585771096988928.0, "train/train_loss": 1.3836020787556966}
+{"train/loss": 0.1225, "train/grad_norm": 0.5834763050079346, "train/learning_rate": 0.0, "train/epoch": 16.0, "train/global_step": 200, "_timestamp": 1710331610.500224, "_runtime": 2787.118215084076, "_step": 24, "train/train_runtime": 1797.7365, "train/train_samples_per_second": 1.78, "train/train_steps_per_second": 0.111, "train/total_flos": 1.0441269033727488e+16, "train/train_loss": 0.25989941954612733}

wandb/run-20240313_112023-t8o614bo/logs/debug-internal.log

@@ -1384,3 +1384,51 @@
 2024-03-13 12:05:26,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
 2024-03-13 12:05:26,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
 2024-03-13 12:05:26,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:05:28,258 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:33,258 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:38,259 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:41,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:05:41,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:05:41,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:05:44,215 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:49,216 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:52,358 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 12:05:54,359 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:05:56,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:05:56,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:05:56,162 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:00,275 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:05,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:10,277 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:11,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:11,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:11,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:16,275 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:21,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:22,359 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 12:06:26,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:26,122 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:26,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:27,275 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:32,276 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:37,277 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:41,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:41,123 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:41,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages
+2024-03-13 12:06:42,295 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:46,399 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 12:06:46,400 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 12:06:46,400 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 12:06:46,400 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 12:06:46,432 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 12:06:47,401 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:50,500 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-13 12:06:50,501 DEBUG   SenderThread:5135 [sender.py:send():378] send: history
+2024-03-13 12:06:50,501 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: summary_record
+2024-03-13 12:06:50,502 INFO    SenderThread:5135 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
+2024-03-13 12:06:51,434 INFO    Thread-12 :5135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240313_112023-t8o614bo/files/wandb-summary.json
+2024-03-13 12:06:52,359 DEBUG   SenderThread:5135 [sender.py:send():378] send: stats
+2024-03-13 12:06:53,360 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: status_report
+2024-03-13 12:06:56,122 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: stop_status
+2024-03-13 12:06:56,123 DEBUG   SenderThread:5135 [sender.py:send_request():405] send_request: stop_status
+2024-03-13 12:06:56,163 DEBUG   HandlerThread:5135 [handler.py:handle_request():146] handle_request: internal_messages