kertob commited on
Commit
1fa12ea
1 Parent(s): 937842a

Training in progress, step 70

Browse files
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6c38dac264184caebde634b60dea7d9ad3b2942a76173ca94775c96d2173b2b2
3
  size 261131840
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4c79267ebfa8a920151c34ada1ab0892488cb5051725b369830b338e72e7665
3
  size 261131840
runs/Mar08_13-48-53_4997a2709b0f/events.out.tfevents.1709905734.4997a2709b0f.35964.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7bec81605b3e0e401e5393389c1106505a374ad704126f36e03f6523a57f7dd8
3
- size 6974
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c45e11e4cdf511edd44d447b868dbfd171f89f42f9ee552866178755eac00313
3
+ size 7181
wandb/debug-internal.log CHANGED
@@ -322,3 +322,49 @@
322
  2024-03-08 13:58:22,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
323
  2024-03-08 13:58:22,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
324
  2024-03-08 13:58:26,249 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
322
  2024-03-08 13:58:22,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
323
  2024-03-08 13:58:22,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
324
  2024-03-08 13:58:26,249 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
325
+ 2024-03-08 13:58:31,251 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
326
+ 2024-03-08 13:58:36,252 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
327
+ 2024-03-08 13:58:37,183 DEBUG SenderThread:37167 [sender.py:send():378] send: stats
328
+ 2024-03-08 13:58:37,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
329
+ 2024-03-08 13:58:37,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
330
+ 2024-03-08 13:58:37,694 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
331
+ 2024-03-08 13:58:41,860 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
332
+ 2024-03-08 13:58:46,862 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
333
+ 2024-03-08 13:58:51,863 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
334
+ 2024-03-08 13:58:52,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
335
+ 2024-03-08 13:58:52,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
336
+ 2024-03-08 13:58:52,694 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
337
+ 2024-03-08 13:58:56,890 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
338
+ 2024-03-08 13:59:01,891 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
339
+ 2024-03-08 13:59:06,892 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
340
+ 2024-03-08 13:59:07,184 DEBUG SenderThread:37167 [sender.py:send():378] send: stats
341
+ 2024-03-08 13:59:07,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
342
+ 2024-03-08 13:59:07,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
343
+ 2024-03-08 13:59:07,695 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
344
+ 2024-03-08 13:59:11,927 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
345
+ 2024-03-08 13:59:16,929 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
346
+ 2024-03-08 13:59:21,930 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
347
+ 2024-03-08 13:59:22,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
348
+ 2024-03-08 13:59:22,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
349
+ 2024-03-08 13:59:22,695 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
350
+ 2024-03-08 13:59:27,345 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
351
+ 2024-03-08 13:59:32,346 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
352
+ 2024-03-08 13:59:37,184 DEBUG SenderThread:37167 [sender.py:send():378] send: stats
353
+ 2024-03-08 13:59:37,655 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
354
+ 2024-03-08 13:59:37,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
355
+ 2024-03-08 13:59:37,695 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
356
+ 2024-03-08 13:59:37,941 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
357
+ 2024-03-08 13:59:42,942 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
358
+ 2024-03-08 13:59:45,430 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: partial_history
359
+ 2024-03-08 13:59:45,431 DEBUG SenderThread:37167 [sender.py:send():378] send: history
360
+ 2024-03-08 13:59:45,432 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: summary_record
361
+ 2024-03-08 13:59:45,432 INFO SenderThread:37167 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
362
+ 2024-03-08 13:59:46,179 INFO Thread-12 :37167 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_134901-hlg2f18u/files/wandb-summary.json
363
+ 2024-03-08 13:59:48,180 INFO Thread-12 :37167 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_134901-hlg2f18u/files/output.log
364
+ 2024-03-08 13:59:48,472 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
365
+ 2024-03-08 13:59:52,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
366
+ 2024-03-08 13:59:52,655 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
367
+ 2024-03-08 13:59:52,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
368
+ 2024-03-08 13:59:53,925 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
369
+ 2024-03-08 13:59:58,926 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
370
+ 2024-03-08 14:00:03,927 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
wandb/run-20240308_134901-hlg2f18u/files/output.log CHANGED
@@ -4,3 +4,4 @@ Checkpoint destination directory ./checkpoint-20 already exists and is non-empty
4
  Checkpoint destination directory ./checkpoint-30 already exists and is non-empty. Saving will proceed but saved results may be invalid.
5
  Checkpoint destination directory ./checkpoint-40 already exists and is non-empty. Saving will proceed but saved results may be invalid.
6
  Checkpoint destination directory ./checkpoint-50 already exists and is non-empty. Saving will proceed but saved results may be invalid.
 
 
4
  Checkpoint destination directory ./checkpoint-30 already exists and is non-empty. Saving will proceed but saved results may be invalid.
5
  Checkpoint destination directory ./checkpoint-40 already exists and is non-empty. Saving will proceed but saved results may be invalid.
6
  Checkpoint destination directory ./checkpoint-50 already exists and is non-empty. Saving will proceed but saved results may be invalid.
7
+ Checkpoint destination directory ./checkpoint-60 already exists and is non-empty. Saving will proceed but saved results may be invalid.
wandb/run-20240308_134901-hlg2f18u/files/wandb-summary.json CHANGED
@@ -1 +1 @@
1
- {"train/loss": 0.3697, "train/grad_norm": 0.572875440120697, "train/learning_rate": 8.2070410981557e-05, "train/epoch": 4.8, "train/global_step": 60, "_timestamp": 1709906293.6760013, "_runtime": 552.2557044029236, "_step": 5}
 
1
+ {"train/loss": 0.3484, "train/grad_norm": 0.6353679299354553, "train/learning_rate": 7.546600811643816e-05, "train/epoch": 5.6, "train/global_step": 70, "_timestamp": 1709906385.4299374, "_runtime": 644.009640455246, "_step": 6}
wandb/run-20240308_134901-hlg2f18u/logs/debug-internal.log CHANGED
@@ -322,3 +322,49 @@
322
  2024-03-08 13:58:22,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
323
  2024-03-08 13:58:22,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
324
  2024-03-08 13:58:26,249 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
322
  2024-03-08 13:58:22,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
323
  2024-03-08 13:58:22,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
324
  2024-03-08 13:58:26,249 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
325
+ 2024-03-08 13:58:31,251 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
326
+ 2024-03-08 13:58:36,252 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
327
+ 2024-03-08 13:58:37,183 DEBUG SenderThread:37167 [sender.py:send():378] send: stats
328
+ 2024-03-08 13:58:37,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
329
+ 2024-03-08 13:58:37,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
330
+ 2024-03-08 13:58:37,694 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
331
+ 2024-03-08 13:58:41,860 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
332
+ 2024-03-08 13:58:46,862 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
333
+ 2024-03-08 13:58:51,863 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
334
+ 2024-03-08 13:58:52,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
335
+ 2024-03-08 13:58:52,654 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
336
+ 2024-03-08 13:58:52,694 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
337
+ 2024-03-08 13:58:56,890 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
338
+ 2024-03-08 13:59:01,891 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
339
+ 2024-03-08 13:59:06,892 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
340
+ 2024-03-08 13:59:07,184 DEBUG SenderThread:37167 [sender.py:send():378] send: stats
341
+ 2024-03-08 13:59:07,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
342
+ 2024-03-08 13:59:07,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
343
+ 2024-03-08 13:59:07,695 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
344
+ 2024-03-08 13:59:11,927 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
345
+ 2024-03-08 13:59:16,929 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
346
+ 2024-03-08 13:59:21,930 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
347
+ 2024-03-08 13:59:22,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
348
+ 2024-03-08 13:59:22,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
349
+ 2024-03-08 13:59:22,695 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
350
+ 2024-03-08 13:59:27,345 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
351
+ 2024-03-08 13:59:32,346 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
352
+ 2024-03-08 13:59:37,184 DEBUG SenderThread:37167 [sender.py:send():378] send: stats
353
+ 2024-03-08 13:59:37,655 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
354
+ 2024-03-08 13:59:37,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
355
+ 2024-03-08 13:59:37,695 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
356
+ 2024-03-08 13:59:37,941 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
357
+ 2024-03-08 13:59:42,942 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
358
+ 2024-03-08 13:59:45,430 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: partial_history
359
+ 2024-03-08 13:59:45,431 DEBUG SenderThread:37167 [sender.py:send():378] send: history
360
+ 2024-03-08 13:59:45,432 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: summary_record
361
+ 2024-03-08 13:59:45,432 INFO SenderThread:37167 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end
362
+ 2024-03-08 13:59:46,179 INFO Thread-12 :37167 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_134901-hlg2f18u/files/wandb-summary.json
363
+ 2024-03-08 13:59:48,180 INFO Thread-12 :37167 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_134901-hlg2f18u/files/output.log
364
+ 2024-03-08 13:59:48,472 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
365
+ 2024-03-08 13:59:52,654 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: stop_status
366
+ 2024-03-08 13:59:52,655 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: internal_messages
367
+ 2024-03-08 13:59:52,655 DEBUG SenderThread:37167 [sender.py:send_request():405] send_request: stop_status
368
+ 2024-03-08 13:59:53,925 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
369
+ 2024-03-08 13:59:58,926 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report
370
+ 2024-03-08 14:00:03,927 DEBUG HandlerThread:37167 [handler.py:handle_request():146] handle_request: status_report